From 0e4cc050d63058969c53cce295dea99e90483c01 Mon Sep 17 00:00:00 2001 From: Sergey Mironov Date: Thu, 24 Oct 2019 18:15:55 +0300 Subject: [PATCH 001/143] Add support for resumable downloads for HTTP protocol. --- transformers/configuration_auto.py | 3 ++ transformers/configuration_utils.py | 7 ++++- transformers/file_utils.py | 47 ++++++++++++++++++++++------- transformers/modeling_auto.py | 8 +++++ transformers/modeling_tf_auto.py | 12 ++++++++ transformers/modeling_tf_utils.py | 8 ++++- transformers/modeling_utils.py | 8 ++++- transformers/tokenization_auto.py | 3 ++ transformers/tokenization_utils.py | 6 +++- 9 files changed, 87 insertions(+), 15 deletions(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a670c..0e8eb03d03 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -92,6 +92,9 @@ class AutoConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index cfa6502bcd..0f9d609b20 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -93,6 +93,9 @@ class PretrainedConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -119,6 +122,7 @@ class PretrainedConfig(object): """ cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) @@ -130,7 +134,8 @@ class PretrainedConfig(object): config_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: - resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_config_archive_map: msg = "Couldn't reach server at '{}' to download pretrained model configuration file.".format( diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 27875212ff..24abd60781 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -22,6 +22,7 @@ from botocore.config import Config from botocore.exceptions import ClientError import requests from tqdm import tqdm +from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -152,7 +153,7 @@ def filename_to_url(filename, cache_dir=None): return url, etag -def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None): +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False): """ Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and @@ -161,6 +162,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N Args: cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). force_download: if True, re-dowload the file even if it's already cached in the cache dir. + resume_download: if True, resume the download if incompletly recieved file is found. """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -173,7 +175,9 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N if parsed.scheme in ('http', 'https', 's3'): # URL, so get it from the cache (downloading if necessary) - return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + return get_from_cache(url_or_filename, cache_dir=cache_dir, + force_download=force_download, proxies=proxies, + resume_download=resume_download) elif os.path.exists(url_or_filename): # File, and it exists. return url_or_filename @@ -234,19 +238,22 @@ def s3_get(url, temp_file, proxies=None): s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) -def http_get(url, temp_file, proxies=None): - req = requests.get(url, stream=True, proxies=proxies) - content_length = req.headers.get('Content-Length') - total = int(content_length) if content_length is not None else None - progress = tqdm(unit="B", total=total) - for chunk in req.iter_content(chunk_size=1024): +def http_get(url, temp_file, proxies=None, resume_size=0): + headers={'Range':'bytes=%d-'%(resume_size,)} if resume_size > 0 else None + response = requests.get(url, stream=True, proxies=proxies, headers=headers) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get('Content-Length') + total = resume_size + int(content_length) if content_length is not None else None + progress = tqdm(unit="B", total=total, initial=resume_size) + for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) temp_file.write(chunk) progress.close() -def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10): +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False): """ Given a URL, look for the corresponding dataset in the local cache. If it's not there, download it. Then return the path to the cached file. @@ -289,17 +296,35 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag if matching_files: cache_path = os.path.join(cache_dir, matching_files[-1]) + if resume_download: + incomplete_path = cache_path + '.incomplete' + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path,'a+b') as f: + yield f + os.remove(incomplete_path) + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = tempfile.NamedTemporaryFile + resume_size = 0 + if not os.path.exists(cache_path) or force_download: # Download to temporary file, then copy to cache dir once finished. # Otherwise you get corrupt cache entries if the download gets interrupted. - with tempfile.NamedTemporaryFile() as temp_file: + with temp_file_manager() as temp_file: logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) # GET file object if url.startswith("s3://"): + if resume_download: + logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') s3_get(url, temp_file, proxies=proxies) else: - http_get(url, temp_file, proxies=proxies) + http_get(url, temp_file, proxies=proxies, resume_size=resume_size) # we are copying the file before closing it, so flush to avoid truncation temp_file.flush() diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..ea5a539c8c 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -112,6 +112,9 @@ class AutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -237,6 +240,8 @@ class AutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. @@ -357,6 +362,9 @@ class AutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index df0ad6e401..cfe19ead2a 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -109,6 +109,9 @@ class TFAutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -237,6 +240,9 @@ class TFAutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -360,6 +366,9 @@ class TFAutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -472,6 +481,9 @@ class TFAutoModelForQuestionAnswering(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index a96e2765fd..aed2a25643 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -176,6 +176,9 @@ class TFPreTrainedModel(tf.keras.Model): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -201,6 +204,7 @@ class TFPreTrainedModel(tf.keras.Model): cache_dir = kwargs.pop('cache_dir', None) from_pt = kwargs.pop('from_pt', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) # Load config @@ -209,6 +213,7 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -236,7 +241,8 @@ class TFPreTrainedModel(tf.keras.Model): # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + resume_download=resume_download, proxies=proxies) except EnvironmentError as e: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: logger.error( diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d082137d5d..db3ab93f8f 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -246,6 +246,9 @@ class PreTrainedModel(nn.Module): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -275,6 +278,7 @@ class PreTrainedModel(nn.Module): cache_dir = kwargs.pop('cache_dir', None) from_tf = kwargs.pop('from_tf', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) output_loading_info = kwargs.pop('output_loading_info', False) @@ -284,6 +288,7 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -315,7 +320,8 @@ class PreTrainedModel(nn.Module): # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: msg = "Couldn't reach server at '{}' to download pretrained weights.".format( diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de17f..455f503a9d 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -87,6 +87,9 @@ class AutoTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 5e5be872ef..b602ebf20a 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -251,6 +251,9 @@ class PreTrainedTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -286,6 +289,7 @@ class PreTrainedTokenizer(object): def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) s3_models = list(cls.max_model_input_sizes.keys()) @@ -352,7 +356,7 @@ class PreTrainedTokenizer(object): if file_path is None: resolved_vocab_files[file_id] = None else: - resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in s3_models: msg = "Couldn't reach server at '{}' to download vocabulary files." From 8d6b9d717c676560d291a7d2913b44e1cf226cd5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 4 Nov 2019 17:07:51 +0100 Subject: [PATCH 002/143] fix #1532 and encode_plus --- .../tests/tokenization_tests_commons.py | 21 ++++-- transformers/tokenization_utils.py | 74 ++++++++++++++++--- 2 files changed, 77 insertions(+), 18 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..cb137e8395 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -223,7 +223,11 @@ class CommonTestCases: sequence = tokenizer.encode(seq_0, add_special_tokens=False) num_added_tokens = tokenizer.num_added_tokens() total_length = len(sequence) + num_added_tokens - information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) + information = tokenizer.encode_plus(seq_0, + max_length=total_length - 2, + add_special_tokens=True, + stride=stride, + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -250,10 +254,12 @@ class CommonTestCases: ) information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, - stride=stride, truncation_strategy='only_second') + stride=stride, truncation_strategy='only_second', + return_overflowing_tokens=True) information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, stride=stride, - truncation_strategy='only_first') + truncation_strategy='only_first', + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -285,7 +291,7 @@ class CommonTestCases: # Testing single inputs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True, return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -297,7 +303,8 @@ class CommonTestCases: # Testing inputs pairs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) + tokenizer.encode(sequence_1, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -309,7 +316,9 @@ class CommonTestCases: # Testing with already existing special tokens if tokenizer.cls_token_id == tokenizer.unk_token_id and tokenizer.cls_token_id == tokenizer.unk_token_id: tokenizer.add_special_tokens({'cls_token': '', 'sep_token': ''}) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, + add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask_orig = encoded_sequence_dict["special_tokens_mask"] special_tokens_mask = tokenizer.get_special_tokens_mask(encoded_sequence_w_special, already_has_special_tokens=True) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index ac765165e2..f860755775 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -744,6 +744,9 @@ class PreTrainedTokenizer(object): stride=0, truncation_strategy='longest_first', return_tensors=None, + return_token_type_ids=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair and additional informations: @@ -770,7 +773,30 @@ class PreTrainedTokenizer(object): - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). **kwargs: passed to the `self.tokenize()` method + + Return: + A Dictionary of shape:: + + { + input_ids: list[int], + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True + } + + With the fields: + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model + + ``overflowing_tokens``: list of overflowing tokens if a max length is specified. + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified + ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added + tokens and 1 specifying sequence tokens. """ def get_input_ids(text): @@ -792,10 +818,17 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - return_tensors=return_tensors) + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask) def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, - truncation_strategy='longest_first', return_tensors=None): + truncation_strategy='longest_first', + return_tensors=None, + return_token_type_ids=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False): """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates @@ -820,21 +853,27 @@ class PreTrainedTokenizer(object): - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). Return: A Dictionary of shape:: { input_ids: list[int], - overflowing_tokens: list[int] if a ``max_length`` is specified, else None - special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True } With the fields: - ``input_ids``: list of tokens to be fed to a model + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model ``overflowing_tokens``: list of overflowing tokens if a max length is specified. - + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added tokens and 1 specifying sequence tokens. """ @@ -843,23 +882,31 @@ class PreTrainedTokenizer(object): len_pair_ids = len(pair_ids) if pair else 0 encoded_inputs = {} + + # Handle max sequence length total_len = len_ids + len_pair_ids + (self.num_added_tokens(pair=pair) if add_special_tokens else 0) if max_length and total_len > max_length: ids, pair_ids, overflowing_tokens = self.truncate_sequences(ids, pair_ids=pair_ids, num_tokens_to_remove=total_len-max_length, truncation_strategy=truncation_strategy, stride=stride) - encoded_inputs["overflowing_tokens"] = overflowing_tokens - encoded_inputs["num_truncated_tokens"] = total_len - max_length + if return_overflowing_tokens: + encoded_inputs["overflowing_tokens"] = overflowing_tokens + encoded_inputs["num_truncated_tokens"] = total_len - max_length + # Handle special_tokens if add_special_tokens: sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) - encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + special_tokens_mask = self.get_special_tokens_mask(ids, pair_ids) else: sequence = ids + pair_ids if pair else ids token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) + special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0)) + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + # Prepare inputs as tensors if asked if return_tensors == 'tf' and is_tf_available(): sequence = tf.constant([sequence]) token_type_ids = tf.constant([token_type_ids]) @@ -870,12 +917,15 @@ class PreTrainedTokenizer(object): logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) encoded_inputs["input_ids"] = sequence - encoded_inputs["token_type_ids"] = token_type_ids + if return_token_type_ids: + encoded_inputs["token_type_ids"] = token_type_ids if max_length and len(encoded_inputs["input_ids"]) > max_length: encoded_inputs["input_ids"] = encoded_inputs["input_ids"][:max_length] - encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] - encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] + if return_token_type_ids: + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] return encoded_inputs From 8e11de0e86124d61ae883f8ce06d71dfaed9b01f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 1 Nov 2019 16:28:32 +0000 Subject: [PATCH 003/143] model forwards can take an inputs_embeds param --- transformers/modeling_gpt2.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 4abe21d22b..6cbed33733 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -370,9 +370,15 @@ class GPT2Model(GPT2PreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].attn.prune_heads(heads) - def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_shape[-1]) + def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None): + if input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if token_type_ids is not None: token_type_ids = token_type_ids.view(-1, input_shape[-1]) if position_ids is not None: @@ -384,8 +390,9 @@ class GPT2Model(GPT2PreTrainedModel): else: past_length = past[0][0].size(-2) if position_ids is None: - position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) # Attention mask. if attention_mask is not None: @@ -419,7 +426,8 @@ class GPT2Model(GPT2PreTrainedModel): else: head_mask = [None] * self.config.n_layer - inputs_embeds = self.wte(input_ids) + if inputs_embeds is None: + inputs_embeds = self.wte(input_ids) position_embeds = self.wpe(position_ids) if token_type_ids is not None: token_type_embeds = self.wte(token_type_ids) @@ -520,14 +528,15 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, past=past, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = transformer_outputs[0] lm_logits = self.lm_head(hidden_states) @@ -623,14 +632,15 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, mc_token_ids=None, lm_labels=None, mc_labels=None): transformer_outputs = self.transformer(input_ids, past=past, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = transformer_outputs[0] From 9eddf44b7a3bc9373d3f1ed08f7ba2c16e8bf39c Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 4 Nov 2019 17:19:15 +0000 Subject: [PATCH 004/143] docstring + check --- transformers/modeling_gpt2.py | 7 ++++++- transformers/modeling_tf_utils.py | 2 +- transformers/modeling_utils.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 6cbed33733..8a11cda084 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -313,6 +313,9 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want to input a probability distribution of tokens rather than actual tokens. """ @add_start_docstrings("The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.", @@ -371,7 +374,9 @@ class GPT2Model(GPT2PreTrainedModel): self.h[layer].attn.prune_heads(heads) def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None): - if input_ids is not None: + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: input_shape = input_ids.size() input_ids = input_ids.view(-1, input_shape[-1]) elif inputs_embeds is not None: diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index a96e2765fd..f626327283 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -35,7 +35,7 @@ class TFPreTrainedModel(tf.keras.Model): r""" Base class for all TF models. :class:`~transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models - as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. + as well as a few methods common to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. Class attributes (overridden by derived classes): - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture. diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 063f52365d..91067a699b 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -53,7 +53,7 @@ class PreTrainedModel(nn.Module): r""" Base class for all models. :class:`~transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models - as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. + as well as a few methods common to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. Class attributes (overridden by derived classes): - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture. From 00337e9687abd7bd3ff55609dc87f92fc4443aac Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 5 Nov 2019 00:39:18 +0000 Subject: [PATCH 005/143] [inputs_embeds] All PyTorch models --- .../adding_a_new_model/modeling_tf_xxx.py | 4 + templates/adding_a_new_model/modeling_xxx.py | 25 +++-- transformers/modeling_bert.py | 95 ++++++++++++------- transformers/modeling_ctrl.py | 33 +++++-- transformers/modeling_distilbert.py | 37 ++++++-- transformers/modeling_gpt2.py | 3 +- transformers/modeling_openai.py | 43 +++++---- transformers/modeling_roberta.py | 47 ++++++--- transformers/modeling_tf_bert.py | 4 + transformers/modeling_tf_ctrl.py | 4 + transformers/modeling_tf_distilbert.py | 4 + transformers/modeling_tf_gpt2.py | 4 + transformers/modeling_tf_openai.py | 4 + transformers/modeling_tf_roberta.py | 4 + transformers/modeling_tf_transfo_xl.py | 4 + transformers/modeling_tf_xlm.py | 4 + transformers/modeling_tf_xlnet.py | 4 + transformers/modeling_transfo_xl.py | 46 ++++++--- transformers/modeling_xlm.py | 61 ++++++++---- transformers/modeling_xlnet.py | 65 +++++++++---- transformers/tests/modeling_common_test.py | 13 +++ 21 files changed, 361 insertions(+), 147 deletions(-) diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py index c661975768..faaece6f19 100644 --- a/templates/adding_a_new_model/modeling_tf_xxx.py +++ b/templates/adding_a_new_model/modeling_tf_xxx.py @@ -255,6 +255,10 @@ XXX_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare Xxx Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index ff64f13f40..138ce70b2c 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -238,6 +238,10 @@ XXX_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare Xxx Model transformer outputting raw hidden-states without any specific head on top.", @@ -295,7 +299,7 @@ class XxxModel(XxxPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) if token_type_ids is None: @@ -449,14 +453,15 @@ class XxxForSequenceClassification(XxxPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): outputs = self.transformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) pooled_output = outputs[1] @@ -520,14 +525,15 @@ class XxxForTokenClassification(XxxPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): outputs = self.transformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] @@ -603,14 +609,15 @@ class XxxForQuestionAnswering(XxxPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None): outputs = self.transformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 6a702c251c..148bc2bd18 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -158,19 +158,26 @@ class BertEmbeddings(nn.Module): self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.dropout = nn.Dropout(config.hidden_dropout_prob) - def forward(self, input_ids, token_type_ids=None, position_ids=None): - seq_length = input_ids.size(1) - if position_ids is None: - position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) - if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] - words_embeddings = self.word_embeddings(input_ids) + seq_length = input_shape[1] + device = input_ids.device if input_ids is not None else inputs_embeds.device + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).expand(input_shape) + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = inputs_embeds + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) embeddings = self.dropout(embeddings) return embeddings @@ -550,6 +557,10 @@ BERT_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. **encoder_hidden_states**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``: Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. @@ -615,8 +626,8 @@ class BertModel(BertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, - head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, + head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None): """ Forward pass on the Model. The model can behave as an encoder (with only self-attention) as well @@ -632,12 +643,23 @@ class BertModel(BertPreTrainedModel): https://arxiv.org/abs/1706.03762 """ + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device + if attention_mask is None: - attention_mask = torch.ones_like(input_ids) + attention_mask = torch.ones(input_shape) if encoder_attention_mask is None: - encoder_attention_mask = torch.ones_like(input_ids) + encoder_attention_mask = torch.ones(input_shape) if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + token_type_ids = torch.zeros(input_shape, dtype=torch.long) # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. @@ -649,8 +671,8 @@ class BertModel(BertPreTrainedModel): # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] if attention_mask.dim() == 2: if self.config.is_decoder: - batch_size, seq_length = input_ids.size() - seq_ids = torch.arange(seq_length, device=input_ids.device) + batch_size, seq_length = input_shape + seq_ids = torch.arange(seq_length, device=device) causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: @@ -689,7 +711,7 @@ class BertModel(BertPreTrainedModel): else: head_mask = [None] * self.config.num_hidden_layers - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + embedding_output = self.embeddings(input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds) encoder_outputs = self.encoder(embedding_output, attention_mask=extended_attention_mask, head_mask=head_mask, @@ -754,14 +776,15 @@ class BertForPreTraining(BertPreTrainedModel): def get_output_embeddings(self): return self.cls.predictions.decoder - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, masked_lm_labels=None, next_sentence_label=None): outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output, pooled_output = outputs[:2] prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output) @@ -829,7 +852,7 @@ class BertForMaskedLM(BertPreTrainedModel): def get_output_embeddings(self): return self.cls.predictions.decoder - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, lm_labels=None, ): outputs = self.bert(input_ids, @@ -837,6 +860,7 @@ class BertForMaskedLM(BertPreTrainedModel): token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask, + inputs_embeds=inputs_embeds, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask) @@ -908,14 +932,15 @@ class BertForNextSentencePrediction(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, next_sentence_label=None): outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) pooled_output = outputs[1] @@ -975,14 +1000,15 @@ class BertForSequenceClassification(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) pooled_output = outputs[1] @@ -1049,8 +1075,8 @@ class BertForMultipleChoice(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): num_choices = input_ids.shape[1] input_ids = input_ids.view(-1, input_ids.size(-1)) @@ -1062,7 +1088,8 @@ class BertForMultipleChoice(BertPreTrainedModel): attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) pooled_output = outputs[1] @@ -1123,14 +1150,15 @@ class BertForTokenClassification(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] @@ -1207,14 +1235,15 @@ class BertForQuestionAnswering(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None): outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 1873040a8e..405e33602a 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -236,6 +236,10 @@ CTRL_INPUTS_DOCSTRING = r""" Inputs: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.", @@ -302,17 +306,26 @@ class CTRLModel(CTRLPreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].attn.prune_heads(heads) - def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_shape[-1]) + def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None): + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if past is None: past_length = 0 past = [None] * len(self.h) else: past_length = past[0][0].size(-2) if position_ids is None: - position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) # Attention mask. if attention_mask is not None: @@ -354,9 +367,10 @@ class CTRLModel(CTRLPreTrainedModel): token_type_embeds = 0 position_ids = position_ids.view(-1, input_shape[-1]) - inputs_embeds = self.w(input_ids) + if inputs_embeds is None: + inputs_embeds = self.w(input_ids) # inputs_embeds = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded - seq_len = input_ids.shape[-1] + seq_len = input_shape[-1] mask = torch.triu(torch.ones(seq_len, seq_len), 1).to(inputs_embeds.device) inputs_embeds *= np.sqrt(self.d_model_size) @@ -455,14 +469,15 @@ class CTRLLMHeadModel(CTRLPreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, past=past, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = transformer_outputs[0] diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index 989b4affae..aca1670852 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -387,6 +387,10 @@ DISTILBERT_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare DistilBERT encoder/transformer outputting raw hidden-states without any specific head on top.", @@ -436,9 +440,18 @@ class DistilBertModel(DistilBertPreTrainedModel): self.transformer.layer[layer].attention.prune_heads(heads) def forward(self, - input_ids, attention_mask=None, head_mask=None): + input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None): + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = torch.ones_like(input_ids) # (bs, seq_length) + attention_mask = torch.ones(input_shape) # (bs, seq_length) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -455,8 +468,9 @@ class DistilBertModel(DistilBertPreTrainedModel): else: head_mask = [None] * self.config.num_hidden_layers - embedding_output = self.embeddings(input_ids) # (bs, seq_length, dim) - tfmr_output = self.transformer(x=embedding_output, + if inputs_embeds is None: + inputs_embeds = self.embeddings(input_ids) # (bs, seq_length, dim) + tfmr_output = self.transformer(x=inputs_embeds, attn_mask=attention_mask, head_mask=head_mask) hidden_state = tfmr_output[0] @@ -514,10 +528,11 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel): def get_output_embeddings(self): return self.vocab_projector - def forward(self, input_ids, attention_mask=None, head_mask=None, masked_lm_labels=None): + def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, masked_lm_labels=None): dlbrt_output = self.distilbert(input_ids=input_ids, attention_mask=attention_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = dlbrt_output[0] # (bs, seq_length, dim) prediction_logits = self.vocab_transform(hidden_states) # (bs, seq_length, dim) prediction_logits = gelu(prediction_logits) # (bs, seq_length, dim) @@ -578,10 +593,11 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, labels=None): distilbert_output = self.distilbert(input_ids=input_ids, attention_mask=attention_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_state = distilbert_output[0] # (bs, seq_len, dim) pooled_output = hidden_state[:, 0] # (bs, dim) pooled_output = self.pre_classifier(pooled_output) # (bs, dim) @@ -652,10 +668,11 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, head_mask=None, start_positions=None, end_positions=None): + def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None): distilbert_output = self.distilbert(input_ids=input_ids, attention_mask=attention_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = distilbert_output[0] # (bs, max_query_len, dim) hidden_states = self.dropout(hidden_states) # (bs, max_query_len, dim) diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 8a11cda084..6812b235f5 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -315,7 +315,8 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. - This is useful if you want to input a probability distribution of tokens rather than actual tokens. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py index 36c1560f1a..246c3ae47b 100644 --- a/transformers/modeling_openai.py +++ b/transformers/modeling_openai.py @@ -322,6 +322,10 @@ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare OpenAI GPT transformer model outputting raw hidden-states without any specific head on top.", @@ -373,14 +377,22 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].attn.prune_heads(heads) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None): + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if position_ids is None: - # This was used when we had a single embedding matrice from position and token embeddings - # start = self.config.vocab_size + self.config.n_special - # end = start + input_ids.size(-1) - # position_ids = torch.arange(start, end, dtype=torch.long, device=input_ids.device) - position_ids = torch.arange(input_ids.size(-1), dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + # Code is different from when we had a single embedding matrice from position and token embeddings + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange(input_shape[-1], dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) # Attention mask. if attention_mask is not None: @@ -413,11 +425,8 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): else: head_mask = [None] * self.config.n_layer - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_ids.size(-1)) - position_ids = position_ids.view(-1, position_ids.size(-1)) - - inputs_embeds = self.tokens_embed(input_ids) + if inputs_embeds is None: + inputs_embeds = self.tokens_embed(input_ids) position_embeds = self.positions_embed(position_ids) if token_type_ids is not None: token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) @@ -495,13 +504,14 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = transformer_outputs[0] lm_logits = self.lm_head(hidden_states) @@ -584,13 +594,14 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, mc_token_ids=None, lm_labels=None, mc_labels=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = transformer_outputs[0] lm_logits = self.lm_head(hidden_states) diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index cbf285fa95..58b86000bb 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -48,16 +48,24 @@ class RobertaEmbeddings(BertEmbeddings): self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size, padding_idx=self.padding_idx) - def forward(self, input_ids, token_type_ids=None, position_ids=None): - seq_length = input_ids.size(1) + def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + device = input_ids.device if input_ids is not None else inputs_embeds.device + if position_ids is None: # Position numbers begin at padding_idx+1. Padding symbols are ignored. # cf. fairseq's `utils.make_positions` - position_ids = torch.arange(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + position_ids = torch.arange(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).expand(input_shape) return super(RobertaEmbeddings, self).forward(input_ids, token_type_ids=token_type_ids, - position_ids=position_ids) + position_ids=position_ids, + inputs_embeds=inputs_embeds) ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in @@ -126,6 +134,10 @@ ROBERTA_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.", @@ -222,13 +234,14 @@ class RobertaForMaskedLM(BertPreTrainedModel): def get_output_embeddings(self): return self.lm_head.decoder - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, masked_lm_labels=None): outputs = self.roberta(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] prediction_scores = self.lm_head(sequence_output) @@ -309,13 +322,14 @@ class RobertaForSequenceClassification(BertPreTrainedModel): self.roberta = RobertaModel(config) self.classifier = RobertaClassificationHead(config) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None): outputs = self.roberta(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] logits = self.classifier(sequence_output) @@ -372,6 +386,10 @@ class RobertaForMultipleChoice(BertPreTrainedModel): Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: Labels for computing the multiple choice classification loss. Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension @@ -415,8 +433,8 @@ class RobertaForMultipleChoice(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, - position_ids=None, head_mask=None): + def forward(self, input_ids=None, token_type_ids=None, attention_mask=None, labels=None, + position_ids=None, head_mask=None, inputs_embeds=None): num_choices = input_ids.shape[1] flat_input_ids = input_ids.view(-1, input_ids.size(-1)) @@ -487,14 +505,15 @@ class RobertaForTokenClassification(BertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): outputs = self.roberta(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index afe9b2946b..a1275db974 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -616,6 +616,10 @@ BERT_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index c8d181548b..dea590c5c5 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -374,6 +374,10 @@ CTRL_INPUTS_DOCSTRING = r""" Inputs: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 188394816e..65acb9e142 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -508,6 +508,10 @@ DISTILBERT_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare DistilBERT encoder/transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index 4188b273ba..50d58a6749 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -408,6 +408,10 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare GPT2 Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index 747c5171fd..18afa85dce 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -389,6 +389,10 @@ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare OpenAI GPT transformer model outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py index 6f34081d3b..32abea659e 100644 --- a/transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -157,6 +157,10 @@ ROBERTA_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare RoBERTa Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index a3e403ce06..ec37aedd74 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -626,6 +626,10 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index 9ac5d28e1f..496d7d72a8 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -530,6 +530,10 @@ XLM_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare XLM Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 8a25be78c1..bb33e45790 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -762,6 +762,10 @@ XLNET_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare XLNet Model transformer outputing raw hidden-states without any specific head on top.", diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py index 25b46cbc2c..a6a82f0dfe 100644 --- a/transformers/modeling_transfo_xl.py +++ b/transformers/modeling_transfo_xl.py @@ -553,6 +553,10 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", @@ -657,12 +661,12 @@ class TransfoXLModel(TransfoXLPreTrainedModel): logger.info("Head pruning is not implemented for Transformer-XL model") pass - def init_mems(self, data): + def init_mems(self, bsz): if self.mem_len > 0: mems = [] param = next(self.parameters()) for i in range(self.n_layer): - empty = torch.zeros(self.mem_len, data.size(1), self.config.d_model, + empty = torch.zeros(self.mem_len, bsz, self.config.d_model, dtype=param.dtype, device=param.device) mems.append(empty) @@ -693,15 +697,22 @@ class TransfoXLModel(TransfoXLPreTrainedModel): return new_mems - def forward(self, input_ids, mems=None, head_mask=None): + def forward(self, input_ids=None, mems=None, head_mask=None, inputs_embeds=None): # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library # so we transpose here from shape [bsz, len] to shape [len, bsz] - input_ids = input_ids.transpose(0, 1).contiguous() + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_ids = input_ids.transpose(0, 1).contiguous() + qlen, bsz = input_ids.size() + elif inputs_embeds is not None: + inputs_embeds = inputs_embeds.transpose(0, 1).contiguous() + qlen, bsz = inputs_embeds.shape[0], inputs_embeds.shape[1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") if mems is None: - mems = self.init_mems(input_ids) - - qlen, bsz = input_ids.size() + mems = self.init_mems(bsz) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -718,7 +729,10 @@ class TransfoXLModel(TransfoXLPreTrainedModel): else: head_mask = [None] * self.n_layer - word_emb = self.word_emb(input_ids) + if inputs_embeds is not None: + word_emb = inputs_embeds + else: + word_emb = self.word_emb(input_ids) mlen = mems[0].size(0) if mems is not None else 0 klen = mlen + qlen @@ -860,14 +874,18 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): def reset_length(self, tgt_len, ext_len, mem_len): self.transformer.reset_length(tgt_len, ext_len, mem_len) - def init_mems(self, data): - return self.transformer.init_mems(data) + def init_mems(self, bsz): + return self.transformer.init_mems(bsz) - def forward(self, input_ids, mems=None, head_mask=None, labels=None): - bsz = input_ids.size(0) - tgt_len = input_ids.size(1) + def forward(self, input_ids=None, mems=None, head_mask=None, inputs_embeds=None, labels=None): + if input_ids is not None: + bsz, tgt_len = input_ids.size(0), input_ids.size(1) + elif inputs_embeds is not None: + bsz, tgt_len = inputs_embeds.size(0), inputs_embeds.size(1) + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") - transformer_outputs = self.transformer(input_ids, mems=mems, head_mask=head_mask) + transformer_outputs = self.transformer(input_ids, mems=mems, head_mask=head_mask, inputs_embeds=inputs_embeds) last_hidden = transformer_outputs[0] pred_hid = last_hidden[:, -tgt_len:] diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py index 80b7d1edbf..257f0da394 100644 --- a/transformers/modeling_xlm.py +++ b/transformers/modeling_xlm.py @@ -311,6 +311,10 @@ XLM_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare XLM Model transformer outputting raw hidden-states without any specific head on top.", @@ -421,14 +425,21 @@ class XLMModel(XLMPreTrainedModel): for layer, heads in heads_to_prune.items(): self.attentions[layer].prune_heads(heads) - def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, - lengths=None, cache=None, head_mask=None): # removed: src_enc=None, src_len=None + def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, + lengths=None, cache=None, head_mask=None, inputs_embeds=None): # removed: src_enc=None, src_len=None + if input_ids is not None: + bs, slen = input_ids.size() + else: + bs, slen = inputs_embeds.size()[:-1] + if lengths is None: - lengths = (input_ids != self.pad_index).sum(dim=1).long() + if input_ids is not None: + lengths = (input_ids != self.pad_index).sum(dim=1).long() + else: + lengths = torch.LongTensor([slen]*bs) # mask = input_ids != self.pad_index # check inputs - bs, slen = input_ids.size() assert lengths.size(0) == bs assert lengths.max().item() <= slen # input_ids = input_ids.transpose(0, 1) # batch size as dimension 0 @@ -442,10 +453,12 @@ class XLMModel(XLMPreTrainedModel): # if self.is_decoder and src_enc is not None: # src_mask = torch.arange(src_len.max(), dtype=torch.long, device=lengths.device) < src_len[:, None] + device = input_ids.device if input_ids is not None else inputs_embeds.device + # position_ids if position_ids is None: - position_ids = input_ids.new((slen,)).long() - position_ids = torch.arange(slen, out=position_ids).unsqueeze(0) + position_ids = torch.arange(slen, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).expand((bs, slen)) else: assert position_ids.size() == (bs, slen) # (slen, bs) # position_ids = position_ids.transpose(0, 1) @@ -471,7 +484,7 @@ class XLMModel(XLMPreTrainedModel): head_mask = [None] * self.n_layers # do not recompute cached elements - if cache is not None: + if cache is not None and input_ids is not None: _slen = slen - cache['slen'] input_ids = input_ids[:, -_slen:] position_ids = position_ids[:, -_slen:] @@ -481,8 +494,10 @@ class XLMModel(XLMPreTrainedModel): attn_mask = attn_mask[:, -_slen:] # embeddings - tensor = self.embeddings(input_ids) - tensor = tensor + self.position_embeddings(position_ids).expand_as(tensor) + if inputs_embeds is None: + inputs_embeds = self.embeddings(input_ids) + + tensor = inputs_embeds + self.position_embeddings(position_ids).expand_as(inputs_embeds) if langs is not None and self.use_lang_emb: tensor = tensor + self.lang_embeddings(langs) if token_type_ids is not None: @@ -624,8 +639,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel): def get_output_embeddings(self): return self.pred_layer.proj - def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, - lengths=None, cache=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, + lengths=None, cache=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, langs=langs, @@ -633,7 +648,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel): position_ids=position_ids, lengths=lengths, cache=cache, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) output = transformer_outputs[0] outputs = self.pred_layer(output, labels) @@ -685,8 +701,8 @@ class XLMForSequenceClassification(XLMPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, - lengths=None, cache=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, + lengths=None, cache=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, langs=langs, @@ -694,7 +710,8 @@ class XLMForSequenceClassification(XLMPreTrainedModel): position_ids=position_ids, lengths=lengths, cache=cache, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) output = transformer_outputs[0] logits = self.sequence_summary(output) @@ -768,8 +785,8 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, - lengths=None, cache=None, head_mask=None, start_positions=None, end_positions=None): + def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, + lengths=None, cache=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, langs=langs, @@ -777,7 +794,8 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): position_ids=position_ids, lengths=lengths, cache=cache, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = transformer_outputs[0] @@ -863,8 +881,8 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, - lengths=None, cache=None, head_mask=None, start_positions=None, end_positions=None, + def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, + lengths=None, cache=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, is_impossible=None, cls_index=None, p_mask=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, @@ -873,7 +891,8 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): position_ids=position_ids, lengths=lengths, cache=cache, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) output = transformer_outputs[0] diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index 0d94cdaf35..658048a660 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -558,6 +558,10 @@ XLNET_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.", @@ -712,19 +716,29 @@ class XLNetModel(XLNetPreTrainedModel): pos_emb = pos_emb.to(next(self.parameters())) return pos_emb - def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - token_type_ids=None, input_mask=None, head_mask=None): + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None): # the original code for XLNet uses shapes [len, bsz] with the batch dimension at the end # but we want a unified interface in the library with the batch size on the first dimension # so we move here the first dimension (batch) to the end - input_ids = input_ids.transpose(0, 1).contiguous() + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_ids = input_ids.transpose(0, 1).contiguous() + qlen, bsz = input_ids.shape[0], input_ids.shape[1] + elif inputs_embeds is not None: + inputs_embeds.transpose(0, 1).contiguous() + qlen, bsz = inputs_embeds.shape[0], inputs_embeds.shape[1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + token_type_ids = token_type_ids.transpose(0, 1).contiguous() if token_type_ids is not None else None input_mask = input_mask.transpose(0, 1).contiguous() if input_mask is not None else None attention_mask = attention_mask.transpose(0, 1).contiguous() if attention_mask is not None else None perm_mask = perm_mask.permute(1, 2, 0).contiguous() if perm_mask is not None else None target_mapping = target_mapping.permute(1, 2, 0).contiguous() if target_mapping is not None else None - qlen, bsz = input_ids.shape[0], input_ids.shape[1] + mlen = mems[0].shape[0] if mems is not None and mems[0] is not None else 0 klen = mlen + qlen @@ -777,7 +791,10 @@ class XLNetModel(XLNetPreTrainedModel): non_tgt_mask = None ##### Word embeddings and prepare h & g hidden states - word_emb_k = self.word_embedding(input_ids) + if inputs_embeds is not None: + word_emb_k = inputs_embeds + else: + word_emb_k = self.word_embedding(input_ids) output_h = self.dropout(word_emb_k) if target_mapping is not None: word_emb_q = self.mask_emb.expand(target_mapping.shape[0], bsz, -1) @@ -924,8 +941,8 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): def get_output_embeddings(self): return self.lm_loss - def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - token_type_ids=None, input_mask=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, mems=mems, @@ -933,7 +950,8 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): target_mapping=target_mapping, token_type_ids=token_type_ids, input_mask=input_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) logits = self.lm_loss(transformer_outputs[0]) @@ -998,8 +1016,8 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - token_type_ids=None, input_mask=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, labels=None): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, mems=mems, @@ -1007,7 +1025,8 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): target_mapping=target_mapping, token_type_ids=token_type_ids, input_mask=input_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) output = transformer_outputs[0] output = self.sequence_summary(output) @@ -1049,6 +1068,10 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: Labels for computing the multiple choice classification loss. Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension @@ -1093,9 +1116,9 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): self.init_weights() - def forward(self, input_ids, token_type_ids=None, input_mask=None, attention_mask=None, + def forward(self, input_ids=None, token_type_ids=None, input_mask=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - labels=None, head_mask=None): + labels=None, head_mask=None, inputs_embeds=None): num_choices = input_ids.shape[1] flat_input_ids = input_ids.view(-1, input_ids.size(-1)) @@ -1106,7 +1129,7 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): transformer_outputs = self.transformer(flat_input_ids, token_type_ids=flat_token_type_ids, input_mask=flat_input_mask, attention_mask=flat_attention_mask, mems=mems, perm_mask=perm_mask, target_mapping=target_mapping, - head_mask=head_mask) + head_mask=head_mask, inputs_embeds=inputs_embeds) output = transformer_outputs[0] @@ -1178,8 +1201,8 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - token_type_ids=None, input_mask=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None): outputs = self.transformer(input_ids, @@ -1189,7 +1212,8 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): target_mapping=target_mapping, token_type_ids=token_type_ids, input_mask=input_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] @@ -1294,8 +1318,8 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - token_type_ids=None, input_mask=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, is_impossible=None, cls_index=None, p_mask=None,): transformer_outputs = self.transformer(input_ids, attention_mask=attention_mask, @@ -1304,7 +1328,8 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): target_mapping=target_mapping, token_type_ids=token_type_ids, input_mask=input_mask, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) hidden_states = transformer_outputs[0] start_logits = self.start_logits(hidden_states, p_mask=p_mask) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index ddc0f9f3de..38b2ceafa4 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -525,6 +525,19 @@ class CommonTestCases: # self.assertTrue(model.transformer.wte.weight.shape, model.lm_head.weight.shape) # self.assertTrue(check_same_values(model.transformer.wte, model.lm_head)) + def test_inputs_embeds(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + input_ids = inputs_dict["input_ids"] + del inputs_dict["input_ids"] + + for model_class in self.all_model_classes: + model = model_class(config) + model.eval() + + wte = model.get_input_embeddings() + inputs_dict["inputs_embeds"] = wte(input_ids) + outputs = model(**inputs_dict) + class GPTModelTester(CommonModelTester): From a44f112fb9cf8b2b5c0c710308820ff18f80a303 Mon Sep 17 00:00:00 2001 From: Clement Date: Tue, 5 Nov 2019 08:48:26 -0500 Subject: [PATCH 006/143] add authors for models --- .github/ISSUE_TEMPLATE/--new-model-addition.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/--new-model-addition.md b/.github/ISSUE_TEMPLATE/--new-model-addition.md index 96fd85269d..456151d2a9 100644 --- a/.github/ISSUE_TEMPLATE/--new-model-addition.md +++ b/.github/ISSUE_TEMPLATE/--new-model-addition.md @@ -17,6 +17,7 @@ assignees: '' * [ ] the model implementation is available: (give details) * [ ] the model weights are available: (give details) +* [ ] who are the authors: (mention them) ## Additional context From d7d36181fdefdabadc53adf51bed4a2680f5880a Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 5 Nov 2019 17:49:01 +0000 Subject: [PATCH 007/143] GPT-2 XL --- docs/source/pretrained_models.rst | 3 +++ transformers/configuration_gpt2.py | 1 + transformers/modeling_gpt2.py | 1 + transformers/tokenization_gpt2.py | 3 +++ 4 files changed, 8 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 43c08228bd..559c81cbb0 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -73,6 +73,9 @@ Here is the full list of the currently provided pretrained models together with | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``gpt2-large`` | | 36-layer, 1280-hidden, 20-heads, 774M parameters. | | | | | OpenAI's Large-sized GPT-2 English model | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``gpt2-xl`` | | 48-layer, 1600-hidden, 25-heads, 1558M parameters. | +| | | | OpenAI's XL-sized GPT-2 English model | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | Transformer-XL | ``transfo-xl-wt103`` | | 18-layer, 1024-hidden, 16-heads, 257M parameters. | | | | | English model trained on wikitext-103 | diff --git a/transformers/configuration_gpt2.py b/transformers/configuration_gpt2.py index e7d853f317..c2fb4948d3 100644 --- a/transformers/configuration_gpt2.py +++ b/transformers/configuration_gpt2.py @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json", "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json", + "gpt2-xl": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-config.json", "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-config.json",} class GPT2Config(PretrainedConfig): diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 6812b235f5..e3d26797c8 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -39,6 +39,7 @@ logger = logging.getLogger(__name__) GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-pytorch_model.bin", "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-pytorch_model.bin", + "gpt2-xl": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-pytorch_model.bin", "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-pytorch_model.bin",} def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path): diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 6a7f75acb2..4bec515903 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -46,6 +46,7 @@ PRETRAINED_VOCAB_FILES_MAP = { 'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json", 'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json", 'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-vocab.json", + 'gpt2-xl': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-vocab.json", 'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-vocab.json", }, 'merges_file': @@ -53,6 +54,7 @@ PRETRAINED_VOCAB_FILES_MAP = { 'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt", 'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt", 'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-merges.txt", + 'gpt2-xl': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-merges.txt", 'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-merges.txt", }, } @@ -61,6 +63,7 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'gpt2': 1024, 'gpt2-medium': 1024, 'gpt2-large': 1024, + 'gpt2-xl': 1024, 'distilgpt2': 1024, } From de890ae67d43e1e5d031a815dab5dfed081e9a95 Mon Sep 17 00:00:00 2001 From: Dom Hudson Date: Tue, 5 Nov 2019 11:04:59 +0000 Subject: [PATCH 008/143] Updating docblocks in optimizers.py --- transformers/optimization.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/transformers/optimization.py b/transformers/optimization.py index 39dc7a50ff..a48b5fea54 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -32,8 +32,9 @@ class ConstantLRSchedule(LambdaLR): class WarmupConstantSchedule(LambdaLR): """ Linear warmup and then constant. - Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps. - Keeps learning rate schedule equal to 1. after warmup_steps. + Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. + Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. + Keeps multiplicative variable equal to 1. after warmup_steps. """ def __init__(self, optimizer, warmup_steps, last_epoch=-1): self.warmup_steps = warmup_steps @@ -47,8 +48,9 @@ class WarmupConstantSchedule(LambdaLR): class WarmupLinearSchedule(LambdaLR): """ Linear warmup and then linear decay. - Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps. - Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps. + Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. + Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. + Linearly decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps. """ def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1): self.warmup_steps = warmup_steps @@ -63,9 +65,10 @@ class WarmupLinearSchedule(LambdaLR): class WarmupCosineSchedule(LambdaLR): """ Linear warmup and then cosine decay. - Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps. - Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve. - If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup. + Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. + Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. + Decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve. + If `cycles` (default=0.5) is different from default, then the multiplicative variable follows cosine function after warmup. """ def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1): self.warmup_steps = warmup_steps @@ -83,8 +86,9 @@ class WarmupCosineSchedule(LambdaLR): class WarmupCosineWithHardRestartsSchedule(LambdaLR): """ Linear warmup and then cosine cycles with hard restarts. - Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps. - If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying + Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. + Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. + If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying learning rate (with hard restarts). """ def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1): From 30968d70afedb1a9815164737cdc3779f2f058fe Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 5 Nov 2019 19:06:12 -0500 Subject: [PATCH 009/143] misc doc --- docs/source/serialization.rst | 2 +- examples/contrib/run_openai_gpt.py | 2 +- templates/adding_a_new_model/README.md | 14 ++++++------ .../adding_a_new_model/modeling_tf_xxx.py | 2 +- templates/adding_a_new_model/modeling_xxx.py | 22 ++++++++++++++----- transformers/modeling_utils.py | 2 +- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index c948104d69..d2862dc0b5 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -106,7 +106,7 @@ This section explain how you can save and re-load a fine-tuned model (BERT, GPT, There are three types of files you need to save to be able to reload a fine-tuned model: -* the model it-self which should be saved following PyTorch serialization `best practices `__\ , +* the model itself which should be saved following PyTorch serialization `best practices `__\ , * the configuration file of the model which is saved as a JSON file, and * the vocabulary (and the merges for the BPE-based models GPT and GPT-2). diff --git a/examples/contrib/run_openai_gpt.py b/examples/contrib/run_openai_gpt.py index 661c1c305b..7eb1b0be76 100644 --- a/examples/contrib/run_openai_gpt.py +++ b/examples/contrib/run_openai_gpt.py @@ -237,7 +237,7 @@ def main(): # Save a trained model if args.do_train: # Save a trained model, configuration and tokenizer - model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self + model_to_save = model.module if hasattr(model, 'module') else model # Only save the model itself # If we save using the predefined names, we can load using `from_pretrained` output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME) diff --git a/templates/adding_a_new_model/README.md b/templates/adding_a_new_model/README.md index 1569b51e89..b546555bd6 100644 --- a/templates/adding_a_new_model/README.md +++ b/templates/adding_a_new_model/README.md @@ -7,7 +7,7 @@ The library is designed to incorporate a variety of models and code bases. As su One important point though is that the library has the following goals impacting the way models are incorporated: - one specific feature of the API is the capability to run the model and tokenizer inline. The tokenization code thus often have to be slightly adapted to allow for running in the python interpreter. -- the package is also designed to be as self-consistent and with a small and reliable set of packages dependencies. In consequence, additional dependencies are usually not allowed when adding a model but can be allowed for the inclusion of a new tokenizer (recent examples of dependencies added for tokenizer specificites includes `sentencepiece` and `sacremoses`). Please make sure to check the existing dependencies when possible before adding a new one. +- the package is also designed to be as self-consistent and with a small and reliable set of packages dependencies. In consequence, additional dependencies are usually not allowed when adding a model but can be allowed for the inclusion of a new tokenizer (recent examples of dependencies added for tokenizer specificities include `sentencepiece` and `sacremoses`). Please make sure to check the existing dependencies when possible before adding a new one. For a quick overview of the library organization, please check the [QuickStart section of the documentation](https://huggingface.co/transformers/quickstart.html). @@ -20,7 +20,7 @@ Here an overview of the general workflow: - [ ] add tests - [ ] finalize -Let's details what should be done at each step +Let's detail what should be done at each step ## Adding model/configuration/tokenization classes @@ -28,16 +28,16 @@ Here is the workflow for adding model/configuration/tokenization classes: - [ ] copy the python files from the present folder to the main folder and rename them, replacing `xxx` with your model name, - [ ] edit the files to replace `XXX` (with various casing) with your model name -- [ ] copy-past or create a simple configuration class for your model in the `configuration_...` file -- [ ] copy-past or create the code for your model in the `modeling_...` files (PyTorch and TF 2.0) -- [ ] copy-past or create a tokenizer class for your model in the `tokenization_...` file +- [ ] copy-paste or create a simple configuration class for your model in the `configuration_...` file +- [ ] copy-paste or create the code for your model in the `modeling_...` files (PyTorch and TF 2.0) +- [ ] copy-paste or create a tokenizer class for your model in the `tokenization_...` file # Adding conversion scripts Here is the workflow for the conversion scripts: - [ ] copy the conversion script (`convert_...`) from the present folder to the main folder. -- [ ] edit this scipt to convert your original checkpoint weights to the current pytorch ones. +- [ ] edit this script to convert your original checkpoint weights to the current pytorch ones. # Adding tests: @@ -58,5 +58,5 @@ You can then finish the addition step by adding imports for your classes in the - [ ] add your models and tokenizer to `pipeline.py` - [ ] add a link to your conversion script in the main conversion utility (currently in `__main__` but will be moved to the `commands` subfolder in the near future) - [ ] edit the PyTorch to TF 2.0 conversion script to add your model in the `convert_pytorch_checkpoint_to_tf2.py` file -- [ ] add a mention of your model in the doc: `README.md` and the documentation it-self at `docs/source/pretrained_models.rst`. +- [ ] add a mention of your model in the doc: `README.md` and the documentation itself at `docs/source/pretrained_models.rst`. - [ ] upload the pretrained weigths, configurations and vocabulary files. diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py index faaece6f19..f1d898b47a 100644 --- a/templates/adding_a_new_model/modeling_tf_xxx.py +++ b/templates/adding_a_new_model/modeling_tf_xxx.py @@ -49,7 +49,7 @@ TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP = { #################################################### # TF 2.0 Models are constructed using Keras imperative API by sub-classing # - tf.keras.layers.Layer for the layers and -# - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model) +# - TFPreTrainedModel for the models (itself a sub-class of tf.keras.Model) #################################################### #################################################### diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index 138ce70b2c..d023f565f5 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -120,7 +120,7 @@ def load_tf_weights_in_xxx(model, config, tf_checkpoint_path): #################################################### # PyTorch Models are constructed by sub-classing # - torch.nn.Module for the layers and -# - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module) +# - PreTrainedModel for the models (itself a sub-class of torch.nn.Module) #################################################### #################################################### @@ -300,10 +300,19 @@ class XxxModel(XxxPreTrainedModel): self.encoder.layer[layer].attention.prune_heads(heads) def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None): + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = torch.ones_like(input_ids) + attention_mask = torch.ones(input_shape) if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + token_type_ids = torch.zeros(input_shape, dtype=torch.long) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] @@ -337,7 +346,7 @@ class XxxModel(XxxPreTrainedModel): ################################## # Replace this with your model code - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + embedding_output = self.embeddings(input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds) encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) sequence_output = encoder_outputs[0] outputs = (sequence_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here @@ -388,14 +397,15 @@ class XxxForMaskedLM(XxxPreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, masked_lm_labels=None): outputs = self.transformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, - head_mask=head_mask) + head_mask=head_mask, + inputs_embeds=inputs_embeds) sequence_output = outputs[0] prediction_scores = self.cls(sequence_output) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 91067a699b..d51eefab58 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -238,7 +238,7 @@ class PreTrainedModel(nn.Module): """ assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" - # Only save the model it-self if we are using distributed training + # Only save the model itself if we are using distributed training model_to_save = self.module if hasattr(self, 'module') else self # Save configuration file From f88c104d8f79e78a98c8ce6c1f4a78db73142eab Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 5 Nov 2019 19:56:43 -0500 Subject: [PATCH 010/143] [run_tf_glue] Add comment for context --- examples/run_tf_glue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 8878ce726e..54282277d2 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -73,6 +73,8 @@ model.save_pretrained('./save/') if TASK == "mrpc": # Load the TensorFlow model in PyTorch for inspection + # This is to demo the interoperability between the two frameworks, you don't have to + # do this in real life (you can run the inference on the TF model). pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True) # Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task From 13d9135fa5ba628b4b40891ee13b73d1880ae78d Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 6 Nov 2019 14:44:00 +0000 Subject: [PATCH 011/143] [tests] get rid of warning cf. https://docs.pytest.org/en/latest/example/simple.html --- transformers/tests/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transformers/tests/conftest.py b/transformers/tests/conftest.py index 841ebc8df9..44cfe3f464 100644 --- a/transformers/tests/conftest.py +++ b/transformers/tests/conftest.py @@ -9,6 +9,10 @@ def pytest_addoption(parser): ) +def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + def pytest_collection_modifyitems(config, items): if config.getoption("--runslow"): # --runslow given in cli: do not skip slow tests From 27e015bd546cc971af2413f36148421b22667ee7 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 6 Nov 2019 15:56:12 +0000 Subject: [PATCH 012/143] [tests] Flag to test on cuda --- transformers/tests/conftest.py | 8 ++++++++ transformers/tests/modeling_bert_test.py | 21 ++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/transformers/tests/conftest.py b/transformers/tests/conftest.py index 44cfe3f464..f809234cd5 100644 --- a/transformers/tests/conftest.py +++ b/transformers/tests/conftest.py @@ -7,6 +7,9 @@ def pytest_addoption(parser): parser.addoption( "--runslow", action="store_true", default=False, help="run slow tests" ) + parser.addoption( + "--use_cuda", action="store_true", default=False, help="run tests on gpu" + ) def pytest_configure(config): @@ -21,3 +24,8 @@ def pytest_collection_modifyitems(config, items): for item in items: if "slow" in item.keywords: item.add_marker(skip_slow) + +@pytest.fixture +def use_cuda(request): + """ Run test on gpu """ + return request.config.getoption("--use_cuda") diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py index 67be910a7e..6c93c9a187 100644 --- a/transformers/tests/modeling_bert_test.py +++ b/transformers/tests/modeling_bert_test.py @@ -35,6 +35,7 @@ else: pytestmark = pytest.mark.skip("Require Torch") +@pytest.mark.usefixtures("use_cuda") class BertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction, @@ -66,6 +67,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): num_labels=3, num_choices=4, scope=None, + device='cpu', ): self.parent = parent self.batch_size = batch_size @@ -89,25 +91,26 @@ class BertModelTest(CommonTestCases.CommonModelTester): self.num_labels = num_labels self.num_choices = num_choices self.scope = scope + self.device = device def prepare_config_and_inputs(self): - input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).to(self.device) input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2).to(self.device) token_type_ids = None if self.use_token_type_ids: - token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size).to(self.device) sequence_labels = None token_labels = None choice_labels = None if self.use_labels: - sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) - token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) - choice_labels = ids_tensor([self.batch_size], self.num_choices) + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size).to(self.device) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels).to(self.device) + choice_labels = ids_tensor([self.batch_size], self.num_choices).to(self.device) config = BertConfig( vocab_size_or_config_json_file=self.vocab_size, @@ -141,6 +144,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertModel(config=config) + model.to(input_ids.device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -309,7 +313,10 @@ class BertModelTest(CommonTestCases.CommonModelTester): def test_config(self): self.config_tester.run_common_tests() - def test_bert_model(self): + def test_bert_model(self, use_cuda=False): + # ^^ This could be a real fixture + if use_cuda: + self.model_tester.device = "cuda" config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) From d5319793c47326655cf25025ceb13a97afa00aad Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 6 Nov 2019 16:21:00 +0000 Subject: [PATCH 013/143] Fix BERT --- transformers/modeling_bert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 148bc2bd18..7c2c6f4602 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -170,7 +170,7 @@ class BertEmbeddings(nn.Module): position_ids = torch.arange(seq_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).expand(input_shape) if token_type_ids is None: - token_type_ids = torch.zeros(input_shape, dtype=torch.long) + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) if inputs_embeds is None: inputs_embeds = self.word_embeddings(input_ids) @@ -655,11 +655,11 @@ class BertModel(BertPreTrainedModel): device = input_ids.device if input_ids is not None else inputs_embeds.device if attention_mask is None: - attention_mask = torch.ones(input_shape) + attention_mask = torch.ones(input_shape, device=device) if encoder_attention_mask is None: - encoder_attention_mask = torch.ones(input_shape) + encoder_attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: - token_type_ids = torch.zeros(input_shape, dtype=torch.long) + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. From 2f3a4210185f5311f6cfab3c91b30616c9a30fc8 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 6 Nov 2019 16:43:09 +0000 Subject: [PATCH 014/143] Fix other PyTorch models --- templates/adding_a_new_model/modeling_xxx.py | 6 ++++-- transformers/modeling_distilbert.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index d023f565f5..1f98c6406f 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -309,10 +309,12 @@ class XxxModel(XxxPreTrainedModel): else: raise ValueError("You have to specify either input_ids or inputs_embeds") + device = input_ids.device if input_ids is not None else inputs_embeds.device + if attention_mask is None: - attention_mask = torch.ones(input_shape) + attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: - token_type_ids = torch.zeros(input_shape, dtype=torch.long) + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index aca1670852..00106627a8 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -450,8 +450,10 @@ class DistilBertModel(DistilBertPreTrainedModel): else: raise ValueError("You have to specify either input_ids or inputs_embeds") + device = input_ids.device if input_ids is not None else inputs_embeds.device + if attention_mask is None: - attention_mask = torch.ones(input_shape) # (bs, seq_length) + attention_mask = torch.ones(input_shape, device=device) # (bs, seq_length) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head From 1c542df7e554a2014051dd09becf60f157fed524 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 6 Nov 2019 16:26:31 -0500 Subject: [PATCH 015/143] Add RoBERTa-based GPT-2 Output Detector from OpenAI converted from https://github.com/openai/gpt-2-output-dataset/tree/master/detector Co-Authored-By: Lysandre Debut Co-Authored-By: Jong Wook Kim Co-Authored-By: Jeff Wu --- docs/source/pretrained_models.rst | 8 ++++++++ transformers/configuration_roberta.py | 2 ++ transformers/modeling_roberta.py | 2 ++ transformers/tokenization_roberta.py | 6 ++++++ 4 files changed, 18 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 559c81cbb0..edb47e7f1c 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -127,6 +127,14 @@ Here is the full list of the currently provided pretrained models together with | | ``roberta-large-mnli`` | | 24-layer, 1024-hidden, 16-heads, 355M parameters | | | | | ``roberta-large`` fine-tuned on `MNLI `__. | | | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``roberta-base-openai-detector`` | | 12-layer, 768-hidden, 12-heads, 125M parameters | +| | | | ``roberta-base`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``roberta-large-openai-detector`` | | 24-layer, 1024-hidden, 16-heads, 355M parameters | +| | | | ``roberta-large`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model. | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | DistilBERT | ``distilbert-base-uncased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | | | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint | diff --git a/transformers/configuration_roberta.py b/transformers/configuration_roberta.py index 367a85211d..842edac56e 100644 --- a/transformers/configuration_roberta.py +++ b/transformers/configuration_roberta.py @@ -29,6 +29,8 @@ ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-config.json", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-config.json", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-config.json", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-config.json", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-config.json", } diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index 58b86000bb..fc27353d37 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -35,6 +35,8 @@ ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-pytorch_model.bin", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-pytorch_model.bin", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-pytorch_model.bin", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-pytorch_model.bin", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-pytorch_model.bin", } class RobertaEmbeddings(BertEmbeddings): diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py index 5e1300fa4d..df3e12bc7c 100644 --- a/transformers/tokenization_roberta.py +++ b/transformers/tokenization_roberta.py @@ -47,6 +47,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-vocab.json", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-vocab.json", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json", }, 'merges_file': { @@ -54,6 +56,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-merges.txt", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-merges.txt", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt", }, } @@ -62,6 +66,8 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'roberta-large': 512, 'roberta-large-mnli': 512, 'distilroberta-base': 512, + 'roberta-base-openai-detector': 512, + 'roberta-large-openai-detector': 512, } From 7a9aae1044aa4699310a8004f631fc0a4bdf1b65 Mon Sep 17 00:00:00 2001 From: Adrian Bauer Date: Thu, 7 Nov 2019 17:08:39 -0500 Subject: [PATCH 016/143] Fix run_bertology.py Make imports and args.overwrite_cache match run_glue.py --- examples/run_bertology.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/run_bertology.py b/examples/run_bertology.py index aae2d150d3..d1d05a1073 100644 --- a/examples/run_bertology.py +++ b/examples/run_bertology.py @@ -39,8 +39,9 @@ from transformers import (WEIGHTS_NAME, from run_glue import set_seed, load_and_cache_examples, ALL_MODELS, MODEL_CLASSES -from utils_glue import (compute_metrics, convert_examples_to_features, - output_modes, processors) +from transformers import glue_compute_metrics as compute_metrics +from transformers import glue_output_modes as output_modes +from transformers import glue_processors as processors logger = logging.getLogger(__name__) @@ -233,6 +234,8 @@ def main(): help="If > 0: limit the data to a subset of data_subset instances.") parser.add_argument("--overwrite_output_dir", action='store_true', help="Whether to overwrite data in output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") parser.add_argument("--dont_normalize_importance_by_layer", action='store_true', help="Don't normalize importance score by layers") From b5d330d11820f4ac2cc8c909b1a6a77e0cd961e0 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 11 Nov 2019 10:15:14 -0500 Subject: [PATCH 017/143] Fix #1784 --- templates/adding_a_new_model/tokenization_xxx.py | 2 +- transformers/tokenization_bert.py | 2 +- transformers/tokenization_roberta.py | 2 +- transformers/tokenization_utils.py | 2 +- transformers/tokenization_xlm.py | 2 +- transformers/tokenization_xlnet.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/adding_a_new_model/tokenization_xxx.py b/templates/adding_a_new_model/tokenization_xxx.py index 1b1325aab5..3d6b4ad9df 100644 --- a/templates/adding_a_new_model/tokenization_xxx.py +++ b/templates/adding_a_new_model/tokenization_xxx.py @@ -172,7 +172,7 @@ class XxxTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index 8affdd9036..ded5072e58 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -220,7 +220,7 @@ class BertTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py index df3e12bc7c..b44e004997 100644 --- a/transformers/tokenization_roberta.py +++ b/transformers/tokenization_roberta.py @@ -120,7 +120,7 @@ class RobertaTokenizer(GPT2Tokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: if token_ids_1 is not None: diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index ac765165e2..cd14cc4582 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -951,7 +951,7 @@ class PreTrainedTokenizer(object): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index d09ce6b9dc..01f8721d98 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -781,7 +781,7 @@ class XLMTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index deae8de336..a4f1a6e3ba 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -208,7 +208,7 @@ class XLNetTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: From 1c7253cc5f233978ddf6804efc71c0358acbac45 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 11 Nov 2019 16:18:16 +0100 Subject: [PATCH 018/143] modeling: add DistilBertForTokenClassification implementation --- transformers/modeling_distilbert.py | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index 00106627a8..d30f493c69 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -30,6 +30,7 @@ import numpy as np import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from .modeling_utils import PreTrainedModel, prune_linear_layer from .configuration_distilbert import DistilBertConfig @@ -702,3 +703,75 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): outputs = (total_loss,) + outputs return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) + + +@add_start_docstrings("""DistilBert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + DISTILBERT_START_DOCSTRING, + DISTILBERT_INPUTS_DOCSTRING) +class DistilBertForTokenClassification(DistilBertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') + model = DistilBertForTokenClassification.from_pretrained('distilbert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + def __init__(self, config): + super(DistilBertForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.distilbert = DistilBertModel(config) + self.dropout = nn.Dropout(config.dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids=None, attention_mask=None, head_mask=None, + inputs_embeds=None, labels=None): + + outputs = self.distilbert(input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels)[active_loss] + active_labels = labels.view(-1)[active_loss] + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), scores, (hidden_states), (attentions) From 1806eabf59fee4a8a79a3a80a927cbb3d0fbde45 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 11 Nov 2019 16:18:48 +0100 Subject: [PATCH 019/143] module: add DistilBertForTokenClassification import --- transformers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index 53f3c39dc7..4f13122f70 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -93,6 +93,7 @@ if is_torch_available(): ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, + DistilBertForTokenClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model From 2b07b9e5ee14ac37fcef7bac958963d869b3b79a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 11 Nov 2019 16:19:34 +0100 Subject: [PATCH 020/143] examples: add DistilBert support for NER fine-tuning --- examples/run_ner.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index b35d8298fe..1c5774df97 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -36,16 +36,18 @@ from utils_ner import convert_examples_to_features, get_labels, read_examples_fr from transformers import AdamW, WarmupLinearSchedule from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer +from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer logger = logging.getLogger(__name__) ALL_MODELS = sum( - (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig)), + (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), ()) MODEL_CLASSES = { "bert": (BertConfig, BertForTokenClassification, BertTokenizer), - "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer) + "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer), + "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer) } @@ -121,9 +123,10 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id): batch = tuple(t.to(args.device) for t in batch) inputs = {"input_ids": batch[0], "attention_mask": batch[1], - "token_type_ids": batch[2] if args.model_type in ["bert", "xlnet"] else None, - # XLM and RoBERTa don"t use segment_ids "labels": batch[3]} + if args.model_type != "distilbert": + inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -206,9 +209,9 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix="" with torch.no_grad(): inputs = {"input_ids": batch[0], "attention_mask": batch[1], - "token_type_ids": batch[2] if args.model_type in ["bert", "xlnet"] else None, - # XLM and RoBERTa don"t use segment_ids "labels": batch[3]} + if args.model_type != "distilbert": + inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] @@ -520,3 +523,4 @@ def main(): if __name__ == "__main__": main() + From 94e55253aef2ccb4b0de95e4aadd6432e3e6a65a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 11 Nov 2019 16:20:15 +0100 Subject: [PATCH 021/143] tests: add test case for DistilBertForTokenClassification implementation --- .../tests/modeling_distilbert_test.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py index 937d03396d..8099c03586 100644 --- a/transformers/tests/modeling_distilbert_test.py +++ b/transformers/tests/modeling_distilbert_test.py @@ -23,6 +23,7 @@ from transformers import is_torch_available if is_torch_available(): from transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, + DistilBertForTokenClassification, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) else: pytestmark = pytest.mark.skip("Require Torch") @@ -180,6 +181,21 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): [self.batch_size, self.num_labels]) self.check_loss_output(result) + def create_and_check_distilbert_for_token_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = DistilBertForTokenClassification(config=config) + model.eval() + + loss, logits = model(input_ids, attention_mask=input_mask, labels=token_labels) + result = { + "loss": loss, + "logits": logits, + } + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.seq_length, self.num_labels]) + self.check_loss_output(result) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs @@ -209,6 +225,10 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs) + def test_for_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_distilbert_for_token_classification(*config_and_inputs) + # @pytest.mark.slow # def test_model_from_pretrained(self): # cache_dir = "/tmp/transformers_test/" From 8aba81a0b64bbf7a2dcd13eaceb543c5f38fd82f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 12 Nov 2019 08:52:43 +0100 Subject: [PATCH 022/143] fix #1789 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 40b08583b1..17dfea6374 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,7 @@ for model_class, tokenizer_class, pretrained_weights in MODELS: # Each architecture is provided with several class for fine-tuning on down-stream tasks, e.g. BERT_MODEL_CLASSES = [BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction, - BertForSequenceClassification, BertForMultipleChoice, BertForTokenClassification, - BertForQuestionAnswering] + BertForSequenceClassification, BertForTokenClassification, BertForQuestionAnswering] # All the classes for an architecture can be initiated from pretrained weights for this architecture # Note that additional weights added for fine-tuning are only initialized From 2e31176557d381171d44b5b51b72411b4c2e0601 Mon Sep 17 00:00:00 2001 From: ronakice Date: Tue, 12 Nov 2019 05:55:11 -0500 Subject: [PATCH 023/143] fix multi-gpu eval --- examples/run_glue.py | 4 ++++ examples/run_lm_finetuning.py | 4 ++++ examples/run_multiple_choice.py | 4 ++++ examples/run_ner.py | 4 ++++ examples/run_squad.py | 4 ++++ examples/run_summarization_finetuning.py | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/examples/run_glue.py b/examples/run_glue.py index 1558a812c3..f82e589301 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -224,6 +224,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu eval + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 2044cfe9e8..d9ee2fdb2b 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -300,6 +300,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu evaluate + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py index 638bbe74f1..c9e13e198d 100644 --- a/examples/run_multiple_choice.py +++ b/examples/run_multiple_choice.py @@ -229,6 +229,10 @@ def evaluate(args, model, tokenizer, prefix="", test=False): eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu evaluate + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) diff --git a/examples/run_ner.py b/examples/run_ner.py index b35d8298fe..c12709e37b 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -191,6 +191,10 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix="" eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu evaluate + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation %s *****", prefix) logger.info(" Num examples = %d", len(eval_dataset)) diff --git a/examples/run_squad.py b/examples/run_squad.py index d9dc2abfde..ad4656462d 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -217,6 +217,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu evaluate + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(dataset)) diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py index 448505c727..f5604c2669 100644 --- a/examples/run_summarization_finetuning.py +++ b/examples/run_summarization_finetuning.py @@ -275,6 +275,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size ) + # multi-gpu evaluate + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) From 7246d3c2f93c4461f3ec8ada7a26a002d8f196ea Mon Sep 17 00:00:00 2001 From: Michael Watkins Date: Wed, 6 Nov 2019 13:18:16 +0200 Subject: [PATCH 024/143] Consider do_lower_case in PreTrainedTokenizer As pointed out in #1545, when using an uncased model, and adding a new uncased token, the tokenizer does not correctly identify this in the case that the input text contains the token in a cased format. For instance, if we load bert-base-uncased into BertTokenizer, and then use .add_tokens() to add "cool-token", we get the expected result for .tokenize('this is a cool-token'). However, we get a possibly unexpected result for .tokenize('this is a cOOl-Token'), which in fact mirrors the result for the former from before the new token was added. This commit adds - functionality to PreTrainedTokenizer to handle this situation in case a tokenizer (currently Bert, DistilBert, and XLNet) has the do_lower_case=True kwarg by: 1) lowercasing tokens added with .add_tokens() 2) lowercasing text at the beginning of .tokenize() - new common test case for tokenizers https://github.com/huggingface/transformers/issues/1545 --- .../tests/tokenization_tests_commons.py | 31 ++++++++++++++++++- transformers/tokenization_utils.py | 5 +++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..287e6fc7b3 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -110,6 +110,36 @@ class CommonTestCases: self.assertListEqual(subwords, subwords_loaded) + def test_added_tokens_do_lower_case(self): + tokenizer = self.get_tokenizer(do_lower_case=True) + + text = "aaaaa bbbbbb low cccccccccdddddddd l" + text2 = "AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l" + + toks0 = tokenizer.tokenize(text) # toks before adding new_toks + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", 'AAAAA BBBBBB', 'CCCCCCCCCDDDDDDDD'] + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 2) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) + self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer + self.assertListEqual(toks, toks2) + + tokenizer = self.get_tokenizer(do_lower_case=False) + + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 4) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) # Length should still be the same + self.assertNotEqual(len(toks), len(toks0)) + self.assertNotEqual(toks[0], toks2[0]) # But at least the first tokens should differ def test_add_tokens_tokenizer(self): tokenizer = self.get_tokenizer() @@ -160,7 +190,6 @@ class CommonTestCases: self.assertEqual(tokens[0], tokenizer.eos_token_id) self.assertEqual(tokens[-2], tokenizer.pad_token_id) - def test_required_methods_tokenizer(self): tokenizer = self.get_tokenizer() input_text, output_text = self.get_input_output_texts() diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cd14cc4582..fc31c10d25 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -512,6 +512,8 @@ class PreTrainedTokenizer(object): to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) + if self.init_kwargs.get('do_lower_case', False): + token = token.lower() if token != self.unk_token and \ self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ token not in to_add_tokens: @@ -605,6 +607,9 @@ class PreTrainedTokenizer(object): Take care of added tokens. """ + if self.init_kwargs.get('do_lower_case', False): + text = text.lower() + def split_on_token(tok, text): result = [] split_text = text.split(tok) From d409aca32632718afbcd098de2bb11b9b71b7df1 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 12 Nov 2019 10:59:37 -0500 Subject: [PATCH 025/143] Clarify the use of past in GPT2 and CTRL --- transformers/modeling_ctrl.py | 9 ++++++--- transformers/modeling_gpt2.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 405e33602a..1ed9e6ebb1 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -220,7 +220,8 @@ CTRL_INPUTS_DOCSTRING = r""" Inputs: **past**: list of ``torch.FloatTensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model - (see `past` output below). Can be used to speed up sequential decoding. + (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -252,7 +253,8 @@ class CTRLModel(CTRLPreTrainedModel): **past**: list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). - Can be used (see `past` input) to speed up sequential decoding. + Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: @@ -437,7 +439,8 @@ class CTRLLMHeadModel(CTRLPreTrainedModel): **past**: list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). - Can be used (see `past` input) to speed up sequential decoding. + Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index e3d26797c8..35bc5c8d6e 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -298,7 +298,8 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: **past**: list of ``torch.FloatTensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model - (see `past` output below). Can be used to speed up sequential decoding. + (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -330,7 +331,8 @@ class GPT2Model(GPT2PreTrainedModel): **past**: list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). - Can be used (see `past` input) to speed up sequential decoding. + Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: @@ -503,7 +505,8 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): **past**: list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). - Can be used (see `past` input) to speed up sequential decoding. + Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: @@ -595,7 +598,8 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): **past**: list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). - Can be used (see `past` input) to speed up sequential decoding. + Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: From dd6b2e05e17d7f3eb1533e9df47f7fd8a1aafb0f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 15:18:09 -0500 Subject: [PATCH 026/143] whitespace --- .gitignore | 3 ++- transformers/modeling_tf_utils.py | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index ec60c045c4..c789666707 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,5 @@ examples/runs serialization_dir # emacs -*.*~ \ No newline at end of file +*.*~ +debug.env diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index f626327283..446fcad131 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -477,10 +477,10 @@ def shape_list(x): return [dynamic[i] if s is None else s for i, s in enumerate(static)] def get_initializer(initializer_range=0.02): - """Creates a `tf.initializers.truncated_normal` with the given range. - Args: - initializer_range: float, initializer range for stddev. - Returns: - TruncatedNormal initializer with stddev = `initializer_range`. - """ - return tf.keras.initializers.TruncatedNormal(stddev=initializer_range) + """Creates a `tf.initializers.truncated_normal` with the given range. + Args: + initializer_range: float, initializer range for stddev. + Returns: + TruncatedNormal initializer with stddev = `initializer_range`. + """ + return tf.keras.initializers.TruncatedNormal(stddev=initializer_range) From 872403be1cd62c9576e0938e2531749fbb6733ea Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 15:22:54 -0500 Subject: [PATCH 027/143] This is not a @property after all --- templates/adding_a_new_model/modeling_xxx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index 1f98c6406f..94c4b0db9a 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -284,7 +284,6 @@ class XxxModel(XxxPreTrainedModel): self.init_weights() - @property def get_input_embeddings(self): return self.embeddings.word_embeddings From 70d97ddd607f3d45e6f7af3ea2f60b3cc7cd13d5 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 16:30:22 -0500 Subject: [PATCH 028/143] [TF models] Common attributes as per #1721 --- transformers/modeling_tf_bert.py | 9 +++++++++ transformers/modeling_tf_ctrl.py | 6 ++++++ transformers/modeling_tf_distilbert.py | 6 ++++++ transformers/modeling_tf_gpt2.py | 9 +++++++++ transformers/modeling_tf_openai.py | 9 +++++++++ transformers/modeling_tf_roberta.py | 6 ++++++ transformers/modeling_tf_transfo_xl.py | 3 +++ transformers/modeling_tf_utils.py | 15 +++++++++++++++ transformers/modeling_tf_xlm.py | 5 +++++ transformers/modeling_tf_xlnet.py | 6 ++++++ transformers/tests/modeling_tf_common_test.py | 10 ++++++++++ 11 files changed, 84 insertions(+) diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index a1275db974..66d5efd87c 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -460,6 +460,9 @@ class TFBertMainLayer(tf.keras.layers.Layer): self.encoder = TFBertEncoder(config, name='encoder') self.pooler = TFBertPooler(config, name='pooler') + def get_input_embeddings(self): + return self.embeddings + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -702,6 +705,9 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.nsp = TFBertNSPHead(config, name='nsp___cls') self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls') + def get_output_embeddings(self): + return self.bert.embeddings + def call(self, inputs, **kwargs): outputs = self.bert(inputs, **kwargs) @@ -747,6 +753,9 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls') + def get_output_embeddings(self): + return self.bert.embeddings + def call(self, inputs, **kwargs): outputs = self.bert(inputs, **kwargs) diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index dea590c5c5..99738a8b14 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -192,6 +192,9 @@ class TFCTRLMainLayer(tf.keras.layers.Layer): name='h_._{}'.format(i)) for i in range(config.n_layer)] self.layernorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name="layernorm") + def get_input_embeddings(self): + return self.w + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -480,6 +483,9 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel): self.lm_head = TFCTRLLMHead(config, self.transformer.w, name="lm_head") + def get_output_embeddings(self): + return self.lm_head.input_embeddings + def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) hidden_states = transformer_outputs[0] diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 65acb9e142..4b1f3e676b 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -398,6 +398,9 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): self.embeddings = TFEmbeddings(config, name="embeddings") # Embeddings self.transformer = TFTransformer(config, name="transformer") # Encoder + def get_input_embeddings(self): + return self.embeddings + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -613,6 +616,9 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): self.vocab_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="vocab_layer_norm") self.vocab_projector = TFDistilBertLMHead(config, self.distilbert.embeddings, name="vocab_projector") + def get_output_embeddings(self): + return self.vocab_projector.input_embeddings + def call(self, inputs, **kwargs): distilbert_output = self.distilbert(inputs, **kwargs) diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index 50d58a6749..23866a1a0a 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -219,6 +219,9 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): name='h_._{}'.format(i)) for i in range(config.n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') + def get_input_embeddings(self): + return self.wte + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -490,6 +493,9 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') + def get_output_embeddings(self): + return self.transformer.wte + def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) hidden_states = transformer_outputs[0] @@ -560,6 +566,9 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): self.transformer = TFGPT2MainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') + def get_output_embeddings(self): + return self.transformer.wte + def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index 18afa85dce..bddd9338b1 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -217,6 +217,9 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer): scale=True, name='h_._{}'.format(i)) for i in range(config.n_layer)] + def get_input_embeddings(self): + return self.tokens_embed + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -462,6 +465,9 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): super(TFOpenAIGPTLMHeadModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') + def get_output_embeddings(self): + return self.transformer.tokens_embed + def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) hidden_states = transformer_outputs[0] @@ -524,6 +530,9 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') + def get_output_embeddings(self): + return self.transformer.tokens_embed + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py index 32abea659e..c335910dc6 100644 --- a/transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -65,6 +65,9 @@ class TFRobertaMainLayer(TFBertMainLayer): super(TFRobertaMainLayer, self).__init__(config, **kwargs) self.embeddings = TFRobertaEmbeddings(config, name='embeddings') + def get_input_embeddings(self): + return self.embeddings + class TFRobertaPreTrainedModel(TFPreTrainedModel): """ An abstract class to handle weights initialization and @@ -280,6 +283,9 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): self.roberta = TFRobertaMainLayer(config, name="roberta") self.lm_head = TFRobertaLMHead(config, self.roberta.embeddings, name="lm_head") + def get_output_embeddings(self): + return self.lm_head.decoder + def call(self, inputs, **kwargs): outputs = self.roberta(inputs, **kwargs) diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index ec37aedd74..8c2a35b352 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -413,6 +413,9 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): name='r_r_bias') super(TFTransfoXLMainLayer, self).build(input_shape) + def get_input_embeddings(self): + return self.word_emb + def _resize_token_embeddings(self, new_num_tokens): return self.word_emb diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 446fcad131..e08605d154 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -65,6 +65,21 @@ class TFPreTrainedModel(tf.keras.Model): # Save config in model self.config = config + def get_input_embeddings(self): + """ Get model's input embeddings + """ + base_model = getattr(self, self.base_model_prefix, self) + if base_model is not self: + return base_model.get_input_embeddings() + else: + raise NotImplementedError + + def get_output_embeddings(self): + """ Get model's output embeddings + Return None if the model doesn't have output embeddings + """ + return None # Overwrite for models with output embeddings + def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): """ Build a resized Embedding Variable from a provided token Embedding Module. Increasing the size will add newly initialized vectors at the end diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index 496d7d72a8..20fbdca732 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -277,6 +277,9 @@ class TFXLMMainLayer(tf.keras.layers.Layer): self.prune_heads({int(layer): list(map(int, heads))}) + def get_input_embeddings(self): + return self.embeddings + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -641,6 +644,8 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): self.transformer = TFXLMMainLayer(config, name='transformer') self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer_._proj') + def get_output_embeddings(self): + return self.pred_layer.input_embeddings def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index bb33e45790..7ab95e7c9f 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -371,6 +371,9 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.layer = [TFXLNetLayer(config, name='layer_._{}'.format(i)) for i in range(config.n_layer)] self.dropout = tf.keras.layers.Dropout(config.dropout) + def get_input_embeddings(self): + return self.word_embedding + def build(self, input_shape): initializer = get_initializer(self.initializer_range) self.mask_emb = self.add_weight(shape=(1, 1, self.d_model), @@ -854,6 +857,9 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): self.transformer = TFXLNetMainLayer(config, name='transformer') self.lm_loss = TFXLNetLMHead(config, self.transformer.word_embedding, name='lm_loss') + def get_output_embeddings(self): + return self.lm_loss.input_embeddings + def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) hidden_state = transformer_outputs[0] diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index f636c42889..0be5fe8e9c 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -360,6 +360,16 @@ class TFCommonTestCases: # self.assertTrue(models_equal) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_embeddings() + assert x is None or instanceof(x, tf.keras.layers.Layer) + + def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() From 9d2398fd99d758126709109e5830c6ffe95606e7 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 16:37:08 -0500 Subject: [PATCH 029/143] Ooopsie --- transformers/tests/modeling_tf_common_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 0be5fe8e9c..ca7beb0925 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -367,7 +367,7 @@ class TFCommonTestCases: model = model_class(config) assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) x = model.get_output_embeddings() - assert x is None or instanceof(x, tf.keras.layers.Layer) + assert x is None or isinstance(x, tf.keras.layers.Layer) def test_tie_model_weights(self): From 2f17464266ef5fe8314f78de1320e16cdf29d909 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 19:56:45 -0500 Subject: [PATCH 030/143] [common attributes] Slightly sharper test coverage --- transformers/tests/modeling_common_test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 38b2ceafa4..777e62459b 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -468,9 +468,15 @@ class CommonTestCases: for model_class in self.all_model_classes: model = model_class(config) - model.get_input_embeddings() + self.assertIsInstance( + model.get_input_embeddings(), + torch.nn.Embedding + ) model.set_input_embeddings(torch.nn.Embedding(10, 10)) - model.get_output_embeddings() + x = model.get_output_embeddings() + self.assertTrue( + x is None or isinstance(x, torch.nn.Linear) + ) def test_tie_model_weights(self): if not self.test_torchscript: From 2aef2f0bbcd3b192af18718684615019a7777a9b Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 20:03:19 -0500 Subject: [PATCH 031/143] [common attributes] Fix previous commit for transfo-xl --- transformers/__init__.py | 1 + transformers/tests/modeling_common_test.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 53f3c39dc7..d922f52a1d 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -72,6 +72,7 @@ if is_torch_available(): OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, + AdaptiveEmbedding, load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel, diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 777e62459b..baf1531403 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -35,7 +35,7 @@ if is_torch_available(): import torch import numpy as np - from transformers import (PretrainedConfig, PreTrainedModel, + from transformers import (AdaptiveEmbedding, PretrainedConfig, PreTrainedModel, BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -470,7 +470,7 @@ class CommonTestCases: model = model_class(config) self.assertIsInstance( model.get_input_embeddings(), - torch.nn.Embedding + (torch.nn.Embedding, AdaptiveEmbedding) ) model.set_input_embeddings(torch.nn.Embedding(10, 10)) x = model.get_output_embeddings() From 155c782a2ccd103cf63ad48a2becd7c76a7d2115 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 11 Nov 2019 22:19:14 -0500 Subject: [PATCH 032/143] [inputs_embeds] All TF models + tests --- transformers/modeling_tf_bert.py | 59 +++++++++++++------ transformers/modeling_tf_ctrl.py | 27 ++++++--- transformers/modeling_tf_distilbert.py | 38 ++++++++---- transformers/modeling_tf_gpt2.py | 44 +++++++++----- transformers/modeling_tf_openai.py | 44 +++++++++----- transformers/modeling_tf_roberta.py | 10 +++- transformers/modeling_tf_transfo_xl.py | 53 +++++++++++------ transformers/modeling_tf_xlm.py | 31 +++++++--- transformers/modeling_tf_xlnet.py | 26 ++++++-- transformers/tests/modeling_tf_bert_test.py | 4 -- transformers/tests/modeling_tf_common_test.py | 21 +++++++ 11 files changed, 252 insertions(+), 105 deletions(-) diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index 66d5efd87c..ad0815e2ca 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -142,19 +142,25 @@ class TFBertEmbeddings(tf.keras.layers.Layer): def _embedding(self, inputs, training=False): """Applies embedding based on inputs tensor.""" - input_ids, position_ids, token_type_ids = inputs + input_ids, position_ids, token_type_ids, inputs_embeds = inputs - seq_length = tf.shape(input_ids)[1] + if input_ids is not None: + input_shape = tf.shape(input_ids) + else: + input_shape = tf.shape(inputs_embeds)[:-1] + + seq_length = input_shape[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) - words_embeddings = tf.gather(self.word_embeddings, input_ids) + if inputs_embeds is None: + inputs_embeds = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = inputs_embeds + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) embeddings = self.dropout(embeddings, training=training) return embeddings @@ -473,28 +479,39 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.shape + elif inputs_embeds is not None: + input_shape = inputs_embeds.shape[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) + attention_mask = tf.fill(input_shape, 1) if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] @@ -523,7 +540,7 @@ class TFBertMainLayer(tf.keras.layers.Layer): head_mask = [None] * self.num_hidden_layers # head_mask = tf.constant([0] * self.num_hidden_layers) - embedding_output = self.embeddings([input_ids, position_ids, token_type_ids], training=training) + embedding_output = self.embeddings([input_ids, position_ids, token_type_ids, inputs_embeds], training=training) encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training) sequence_output = encoder_outputs[0] @@ -901,33 +918,39 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): kernel_initializer=get_initializer(config.initializer_range), name='classifier') - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs - num_choices = tf.shape(input_ids)[1] - seq_length = tf.shape(input_ids)[2] + if input_ids is not None: + num_choices = tf.shape(input_ids)[1] + seq_length = tf.shape(input_ids)[2] + else: + num_choices = tf.shape(inputs_embeds)[1] + seq_length = tf.shape(inputs_embeds)[2] - flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None - flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask, inputs_embeds] outputs = self.bert(flat_inputs, training=training) diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index 99738a8b14..ae66dbc82c 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -204,7 +204,7 @@ class TFCTRLMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] past = inputs[1] if len(inputs) > 1 else past @@ -212,7 +212,8 @@ class TFCTRLMainLayer(tf.keras.layers.Layer): token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids position_ids = inputs[4] if len(inputs) > 4 else position_ids head_mask = inputs[5] if len(inputs) > 5 else head_mask - assert len(inputs) <= 6, "Too many inputs." + inputs_embeds = inputs[6] if len(inputs) > 6 else inputs_embeds + assert len(inputs) <= 7, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') past = inputs.get('past', past) @@ -220,12 +221,20 @@ class TFCTRLMainLayer(tf.keras.layers.Layer): token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 6, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 7, "Too many inputs." else: input_ids = inputs - input_shape = shape_list(input_ids) - input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") if past is None: past_length = 0 @@ -233,8 +242,8 @@ class TFCTRLMainLayer(tf.keras.layers.Layer): else: past_length = shape_list(past[0][0])[-2] if position_ids is None: - position_ids = tf.range(past_length, shape_list(input_ids)[-1] + past_length, dtype=tf.int32)[tf.newaxis, :] - position_ids = tf.tile(position_ids, [shape_list(input_ids)[0], 1]) + position_ids = tf.range(past_length, input_shape[-1] + past_length, dtype=tf.int32)[tf.newaxis, :] + position_ids = tf.tile(position_ids, [input_shape[0], 1]) # Attention mask. if attention_mask is not None: @@ -273,8 +282,8 @@ class TFCTRLMainLayer(tf.keras.layers.Layer): token_type_embeds = 0 position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) - inputs_embeds = self.w(input_ids, mode='embedding') - # x = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded + if inputs_embeds is None: + inputs_embeds = self.w(input_ids, mode='embedding') seq_len = input_shape[-1] mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0) diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 4b1f3e676b..6d393bb95d 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -96,7 +96,7 @@ class TFEmbeddings(tf.keras.layers.Layer): initializer=get_initializer(self.initializer_range)) super(TFEmbeddings, self).build(input_shape) - def call(self, inputs, mode="embedding", training=False): + def call(self, inputs, inputs_embeds=None, mode="embedding", training=False): """Get token embeddings of inputs. Args: inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) @@ -112,13 +112,13 @@ class TFEmbeddings(tf.keras.layers.Layer): https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 """ if mode == "embedding": - return self._embedding(inputs, training=training) + return self._embedding(inputs, inputs_embeds=inputs_embeds, training=training) elif mode == "linear": return self._linear(inputs) else: raise ValueError("mode {} is not valid.".format(mode)) - def _embedding(self, inputs, training=False): + def _embedding(self, inputs, inputs_embeds=None, training=False): """ Parameters ---------- @@ -136,14 +136,19 @@ class TFEmbeddings(tf.keras.layers.Layer): else: input_ids, position_ids = inputs - seq_length = tf.shape(input_ids)[1] + if input_ids is not None: + seq_length = tf.shape(input_ids)[1] + else: + seq_length = tf.shape(inputs_embeds)[1] + if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] - word_embeddings = tf.gather(self.word_embeddings, input_ids) + if inputs_embeds is None: + inputs_embeds = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) # (bs, max_seq_length, dim) - embeddings = word_embeddings + position_embeddings # (bs, max_seq_length, dim) + embeddings = inputs_embeds + position_embeddings # (bs, max_seq_length, dim) embeddings = self.LayerNorm(embeddings) # (bs, max_seq_length, dim) embeddings = self.dropout(embeddings, training=training) # (bs, max_seq_length, dim) return embeddings @@ -407,22 +412,33 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): def _prune_heads(self, heads_to_prune): raise NotImplementedError - def call(self, inputs, attention_mask=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask head_mask = inputs[2] if len(inputs) > 2 else head_mask - assert len(inputs) <= 3, "Too many inputs." + inputs_embeds = inputs[3] if len(inputs) > 3 else inputs_embeds + assert len(inputs) <= 4, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 3, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 4, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = tf.ones(shape_list(input_ids)) # (bs, seq_length) + attention_mask = tf.ones(input_shape) # (bs, seq_length) attention_mask = tf.cast(attention_mask, dtype=tf.float32) # Prepare head mask if needed @@ -435,7 +451,7 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): else: head_mask = [None] * self.num_hidden_layers - embedding_output = self.embeddings(input_ids) # (bs, seq_length, dim) + embedding_output = self.embeddings(input_ids, inputs_embeds=inputs_embeds) # (bs, seq_length, dim) tfmr_output = self.transformer([embedding_output, attention_mask, head_mask], training=training) return tfmr_output # last-layer hidden-state, (all hidden_states), (all attentions) diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index 23866a1a0a..5e416a5e3a 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -231,7 +231,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] past = inputs[1] if len(inputs) > 1 else past @@ -239,7 +239,8 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids position_ids = inputs[4] if len(inputs) > 4 else position_ids head_mask = inputs[5] if len(inputs) > 5 else head_mask - assert len(inputs) <= 6, "Too many inputs." + inputs_embeds = inputs[6] if len(inputs) > 6 else inputs_embeds + assert len(inputs) <= 7, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') past = inputs.get('past', past) @@ -247,17 +248,28 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 6, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 7, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if past is None: past_length = 0 past = [None] * len(self.h) else: past_length = shape_list(past[0][0])[-2] if position_ids is None: - position_ids = tf.range(past_length, shape_list(input_ids)[-1] + past_length, dtype=tf.int32)[tf.newaxis, :] + position_ids = tf.range(past_length, input_shape[-1] + past_length, dtype=tf.int32)[tf.newaxis, :] if attention_mask is not None: # We create a 3D attention mask from a 2D tensor mask. @@ -289,11 +301,10 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): head_mask = [None] * self.num_hidden_layers # head_mask = tf.constant([0] * self.num_hidden_layers) - input_shape = shape_list(input_ids) - input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) - inputs_embeds = self.wte(input_ids, mode='embedding') + if inputs_embeds is None: + inputs_embeds = self.wte(input_ids, mode='embedding') position_embeds = self.wpe(position_ids) if token_type_ids is not None: token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]]) @@ -569,7 +580,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): def get_output_embeddings(self): return self.transformer.wte - def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): + def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, mc_token_ids=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] past = inputs[1] if len(inputs) > 1 else past @@ -577,8 +588,9 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids position_ids = inputs[4] if len(inputs) > 4 else position_ids head_mask = inputs[5] if len(inputs) > 5 else head_mask - mc_token_ids = inputs[6] if len(inputs) > 6 else mc_token_ids - assert len(inputs) <= 7, "Too many inputs." + inputs_embeds = inputs[6] if len(inputs) > 6 else inputs_embeds + mc_token_ids = inputs[7] if len(inputs) > 7 else mc_token_ids + assert len(inputs) <= 8, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') past = inputs.get('past', past) @@ -586,21 +598,25 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) mc_token_ids = inputs.get('mc_token_ids', mc_token_ids) - assert len(inputs) <= 7, "Too many inputs." + assert len(inputs) <= 8, "Too many inputs." else: input_ids = inputs - input_shapes = shape_list(input_ids) + if input_ids is not None: + input_shapes = shape_list(input_ids) + else: + input_shapes = shape_list(inputs_embeds)[:-1] seq_length = input_shapes[-1] - flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None - flat_inputs = [flat_input_ids, past, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + flat_inputs = [flat_input_ids, past, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask, inputs_embeds] transformer_outputs = self.transformer(flat_inputs, training=training) hidden_states = transformer_outputs[0] diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index bddd9338b1..c553d92317 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -229,26 +229,38 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if position_ids is None: - position_ids = tf.range(shape_list(input_ids)[-1], dtype=tf.int32)[tf.newaxis, :] + position_ids = tf.range(input_shape[-1], dtype=tf.int32)[tf.newaxis, :] if attention_mask is not None: # We create a 3D attention mask from a 2D tensor mask. @@ -280,11 +292,10 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer): head_mask = [None] * self.num_hidden_layers # head_mask = tf.constant([0] * self.num_hidden_layers) - input_shape = shape_list(input_ids) - input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) - inputs_embeds = self.tokens_embed(input_ids, mode='embedding') + if inputs_embeds is None: + inputs_embeds = self.tokens_embed(input_ids, mode='embedding') position_embeds = self.positions_embed(position_ids) if token_type_ids is not None: token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]]) @@ -533,36 +544,41 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): def get_output_embeddings(self): return self.transformer.tokens_embed - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, mc_token_ids=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - mc_token_ids = inputs[5] if len(inputs) > 5 else mc_token_ids - assert len(inputs) <= 6, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + mc_token_ids = inputs[6] if len(inputs) > 6 else mc_token_ids + assert len(inputs) <= 7, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) mc_token_ids = inputs.get('mc_token_ids', mc_token_ids) - assert len(inputs) <= 6, "Too many inputs." + assert len(inputs) <= 7, "Too many inputs." else: input_ids = inputs - input_shapes = shape_list(input_ids) + if input_ids is not None: + input_shapes = shape_list(input_ids) + else: + input_shapes = shape_list(inputs_embeds)[:-1] seq_length = input_shapes[-1] - flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None - flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask, inputs_embeds] transformer_outputs = self.transformer(flat_inputs, training=training) hidden_states = transformer_outputs[0] diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py index c335910dc6..450c0c72f2 100644 --- a/transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -48,13 +48,17 @@ class TFRobertaEmbeddings(TFBertEmbeddings): def _embedding(self, inputs, training=False): """Applies embedding based on inputs tensor.""" - input_ids, position_ids, token_type_ids = inputs + input_ids, position_ids, token_type_ids, inputs_embeds = inputs + + if input_ids is not None: + seq_length = tf.shape(input_ids)[1] + else: + seq_length = tf.shape(inputs_embeds)[1] - seq_length = tf.shape(input_ids)[1] if position_ids is None: position_ids = tf.range(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=tf.int32)[tf.newaxis, :] - return super(TFRobertaEmbeddings, self)._embedding([input_ids, position_ids, token_type_ids], training=training) + return super(TFRobertaEmbeddings, self)._embedding([input_ids, position_ids, token_type_ids, inputs_embeds], training=training) class TFRobertaMainLayer(TFBertMainLayer): diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index 8c2a35b352..8a8d11cfbc 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -430,11 +430,11 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): def _prune_heads(self, heads): raise NotImplementedError - def init_mems(self, data): + def init_mems(self, bsz): if self.mem_len > 0: mems = [] for i in range(self.n_layer): - empty = tf.zeros([self.mem_len, shape_list(data)[1], self.d_model]) + empty = tf.zeros([self.mem_len, bsz, self.d_model]) mems.append(empty) return mems @@ -464,28 +464,37 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): return new_mems - def call(self, inputs, mems=None, head_mask=None, training=False): + def call(self, inputs, mems=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] mems = inputs[1] if len(inputs) > 1 else mems head_mask = inputs[2] if len(inputs) > 2 else head_mask - assert len(inputs) <= 3, "Too many inputs." + inputs_embeds = inputs[3] if len(inputs) > 3 else inputs_embeds + assert len(inputs) <= 4, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') mems = inputs.get('mems', mems) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 3, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 4, "Too many inputs." else: input_ids = inputs # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library # so we transpose here from shape [bsz, len] to shape [len, bsz] - input_ids = tf.transpose(input_ids, perm=(1, 0)) + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_ids = tf.transpose(input_ids, perm=(1, 0)) + qlen, bsz = shape_list(input_ids) + elif inputs_embeds is not None: + inputs_embeds = tf.transpose(inputs_embeds, perm=(1, 0, 2)) + qlen, bsz = shape_list(inputs_embeds)[:2] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") if mems is None: - mems = self.init_mems(input_ids) - - qlen, bsz = shape_list(input_ids) + mems = self.init_mems(bsz) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -497,7 +506,10 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): else: head_mask = [None] * self.n_layer - word_emb = self.word_emb(input_ids) + if inputs_embeds is not None: + word_emb = inputs_embeds + else: + word_emb = self.word_emb(input_ids) mlen = shape_list(mems[0])[0] if mems is not None else 0 klen = mlen + qlen @@ -723,28 +735,33 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): def reset_length(self, tgt_len, ext_len, mem_len): self.transformer.reset_length(tgt_len, ext_len, mem_len) - def init_mems(self, data): - return self.transformer.init_mems(data) + def init_mems(self, bsz): + return self.transformer.init_mems(bsz) - def call(self, inputs, mems=None, head_mask=None, labels=None, training=False): + def call(self, inputs, mems=None, head_mask=None, inputs_embeds=None, labels=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] mems = inputs[1] if len(inputs) > 1 else mems head_mask = inputs[2] if len(inputs) > 2 else head_mask - labels = inputs[3] if len(inputs) > 3 else labels - assert len(inputs) <= 4, "Too many inputs." + inputs_embeds = inputs[3] if len(inputs) > 3 else inputs_embeds + labels = inputs[4] if len(inputs) > 4 else labels + assert len(inputs) <= 5, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') mems = inputs.get('mems', mems) head_mask = inputs.get('head_mask', head_mask) + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) labels = inputs.get('labels', labels) - assert len(inputs) <= 4, "Too many inputs." + assert len(inputs) <= 5, "Too many inputs." else: input_ids = inputs - bsz, tgt_len = shape_list(input_ids)[:2] + if input_ids is not None: + bsz, tgt_len = shape_list(input_ids)[:2] + else: + bsz, tgt_len = shape_list(inputs_embeds)[:2] - transformer_outputs = self.transformer([input_ids, mems, head_mask], training=training) + transformer_outputs = self.transformer([input_ids, mems, head_mask, inputs_embeds], training=training) last_hidden = transformer_outputs[0] pred_hid = last_hidden[:, -tgt_len:] diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index 20fbdca732..6f11b0537d 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -291,7 +291,7 @@ class TFXLMMainLayer(tf.keras.layers.Layer): raise NotImplementedError def call(self, inputs, attention_mask=None, langs=None, token_type_ids=None, - position_ids=None, lengths=None, cache=None, head_mask=None, + position_ids=None, lengths=None, cache=None, head_mask=None, inputs_embeds=None, training=False): # removed: src_enc=None, src_len=None if isinstance(inputs, (tuple, list)): input_ids = inputs[0] @@ -302,7 +302,8 @@ class TFXLMMainLayer(tf.keras.layers.Layer): lengths = inputs[5] if len(inputs) > 5 else lengths cache = inputs[6] if len(inputs) > 6 else cache head_mask = inputs[7] if len(inputs) > 7 else head_mask - assert len(inputs) <= 8, "Too many inputs." + inputs_embeds = inputs[8] if len(inputs) > 8 else inputs_embeds + assert len(inputs) <= 9, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) @@ -312,16 +313,28 @@ class TFXLMMainLayer(tf.keras.layers.Layer): lengths = inputs.get('lengths', lengths) cache = inputs.get('cache', cache) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 8, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 9, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + bs, slen = shape_list(input_ids) + elif inputs_embeds is not None: + bs, slen = shape_list(inputs_embeds)[:2] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if lengths is None: - lengths = tf.reduce_sum(tf.cast(tf.not_equal(input_ids, self.pad_index), dtype=tf.int32), axis=1) + if input_ids is not None: + lengths = tf.reduce_sum(tf.cast(tf.not_equal(input_ids, self.pad_index), dtype=tf.int32), axis=1) + else: + lengths = tf.convert_to_tensor([slen]*bs, tf.int32) # mask = input_ids != self.pad_index # check inputs - bs, slen = shape_list(input_ids) # assert shape_list(lengths)[0] == bs tf.debugging.assert_equal(shape_list(lengths)[0], bs) # assert lengths.max().item() <= slen @@ -361,7 +374,7 @@ class TFXLMMainLayer(tf.keras.layers.Layer): head_mask = [None] * self.n_layers # do not recompute cached elements - if cache is not None: + if cache is not None and input_ids is not None: _slen = slen - cache['slen'] input_ids = input_ids[:, -_slen:] position_ids = position_ids[:, -_slen:] @@ -371,8 +384,10 @@ class TFXLMMainLayer(tf.keras.layers.Layer): attn_mask = attn_mask[:, -_slen:] # embeddings - tensor = self.embeddings(input_ids) - tensor = tensor + self.position_embeddings(position_ids) + if inputs_embeds is None: + inputs_embeds = self.embeddings(input_ids) + + tensor = inputs_embeds + self.position_embeddings(position_ids) if langs is not None and self.use_lang_emb: tensor = tensor + self.lang_embeddings(langs) if token_type_ids is not None: diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 7ab95e7c9f..4733ea8589 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -487,7 +487,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): return pos_emb def call(self, inputs, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, - token_type_ids=None, input_mask=None, head_mask=None, training=False): + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask @@ -497,7 +497,8 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): token_type_ids = inputs[5] if len(inputs) > 5 else token_type_ids input_mask = inputs[6] if len(inputs) > 6 else input_mask head_mask = inputs[7] if len(inputs) > 7 else head_mask - assert len(inputs) <= 8, "Too many inputs." + inputs_embeds = inputs[8] if len(inputs) > 8 else inputs_embeds + assert len(inputs) <= 9, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) @@ -507,7 +508,8 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): token_type_ids = inputs.get('token_type_ids', token_type_ids) input_mask = inputs.get('input_mask', input_mask) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 8, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 9, "Too many inputs." else: input_ids = inputs @@ -515,14 +517,23 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): # but we want a unified interface in the library with the batch size on the first dimension # so we move here the first dimension (batch) to the end - input_ids = tf.transpose(input_ids, perm=(1, 0)) + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_ids = tf.transpose(input_ids, perm=(1, 0)) + qlen, bsz = shape_list(input_ids)[:2] + elif inputs_embeds is not None: + inputs_embeds = tf.transpose(inputs_embeds, perm=(1, 0, 2)) + qlen, bsz = shape_list(inputs_embeds)[:2] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + token_type_ids = tf.transpose(token_type_ids, perm=(1, 0)) if token_type_ids is not None else None input_mask = tf.transpose(input_mask, perm=(1, 0)) if input_mask is not None else None attention_mask = tf.transpose(attention_mask, perm=(1, 0)) if attention_mask is not None else None perm_mask = tf.transpose(perm_mask, perm=(1, 2, 0)) if perm_mask is not None else None target_mapping = tf.transpose(target_mapping, perm=(1, 2, 0)) if target_mapping is not None else None - qlen, bsz = shape_list(input_ids)[:2] mlen = shape_list(mems[0])[0] if mems is not None and mems[0] is not None else 0 klen = mlen + qlen @@ -573,7 +584,10 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): non_tgt_mask = None ##### Word embeddings and prepare h & g hidden states - word_emb_k = self.word_embedding(input_ids) + if inputs_embeds is not None: + word_emb_k = inputs_embeds + else: + word_emb_k = self.word_embedding(input_ids) output_h = self.dropout(word_emb_k, training=training) if target_mapping is not None: word_emb_q = tf.tile(self.mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) diff --git a/transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py index a1715d2568..bcee97435e 100644 --- a/transformers/tests/modeling_tf_bert_test.py +++ b/transformers/tests/modeling_tf_bert_test.py @@ -131,10 +131,6 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = TFBertModel(config=config) - # inputs = {'input_ids': input_ids, - # 'attention_mask': input_mask, - # 'token_type_ids': token_type_ids} - # sequence_output, pooled_output = model(**inputs) inputs = {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids} diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index ca7beb0925..2bb7cc9c5f 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -411,6 +411,27 @@ class TFCommonTestCases: first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0] self.assertTrue(tf.math.equal(first, second).numpy().all()) + def test_inputs_embeds(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + input_ids = inputs_dict["input_ids"] + del inputs_dict["input_ids"] + + for model_class in self.all_model_classes: + model = model_class(config) + + wte = model.get_input_embeddings() + try: + x = wte(input_ids, mode="embedding") + except: + try: + x = wte([input_ids], mode="embedding") + except: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) + # ^^ In our TF models, the input_embeddings can take slightly different forms, + # so we try two of them and fall back to just synthetically creating a dummy tensor of ones. + inputs_dict["inputs_embeds"] = x + outputs = model(inputs_dict) + def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): """Creates a random int32 tensor of the shape within the vocab size.""" From 74d0bcb6ff692dbaa52da1fdc2b80ece06f5fbe5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 12 Nov 2019 15:27:57 -0500 Subject: [PATCH 033/143] Fix special tokens addition in decoder --- .../tests/tokenization_tests_commons.py | 20 +++++++++++++++++++ transformers/tokenization_utils.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..fdaf8cc137 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -160,6 +160,26 @@ class CommonTestCases: self.assertEqual(tokens[0], tokenizer.eos_token_id) self.assertEqual(tokens[-2], tokenizer.pad_token_id) + def test_add_special_tokens(self): + tokenizer = self.get_tokenizer() + input_text, output_text = self.get_input_output_texts() + + special_token = "[SPECIAL TOKEN]" + + tokenizer.add_special_tokens({"cls_token": special_token}) + encoded_special_token = tokenizer.encode(special_token, add_special_tokens=False) + assert len(encoded_special_token) == 1 + + text = " ".join([input_text, special_token, output_text]) + encoded = tokenizer.encode(text, add_special_tokens=False) + + input_encoded = tokenizer.encode(input_text, add_special_tokens=False) + output_encoded = tokenizer.encode(output_text, add_special_tokens=False) + special_token_id = tokenizer.encode(special_token, add_special_tokens=False) + assert encoded == input_encoded + special_token_id + output_encoded + + decoded = tokenizer.decode(encoded, skip_special_tokens=True) + assert special_token not in decoded def test_required_methods_tokenizer(self): tokenizer = self.get_tokenizer() diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cd14cc4582..f37f6f3206 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -1055,7 +1055,7 @@ class PreTrainedTokenizer(object): class attributes (cls_token, unk_token...). """ all_toks = self.all_special_tokens - all_ids = list(self._convert_token_to_id(t) for t in all_toks) + all_ids = self.convert_tokens_to_ids(all_toks) return all_ids @staticmethod From 7627dde1f8888faf5b05a8f1dbbc2271096cc1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0brahim=20Ethem=20Demirci?= Date: Thu, 14 Nov 2019 17:06:15 +0300 Subject: [PATCH 034/143] sum() is the leanest method to flatten a string list, so it's been replaced by itertools.chain.from_iterable() --- transformers/tokenization_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cd14cc4582..1eebae08a7 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -21,6 +21,7 @@ import os import json import six import copy +import itertools from io import open from .file_utils import cached_path, is_tf_available, is_torch_available @@ -641,9 +642,9 @@ class PreTrainedTokenizer(object): tokenized_text += [sub_text] text_list = tokenized_text - return sum((self._tokenize(token, **kwargs) if token not \ + return list(itertools.chain.from_iterable((self._tokenize(token, **kwargs) if token not \ in self.added_tokens_encoder and token not in self.all_special_tokens \ - else [token] for token in tokenized_text), []) + else [token] for token in tokenized_text))) added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens tokenized_text = split_on_tokens(added_tokens, text) From 022525b0031bcdbbb62d1223f75919983f2ac426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Tue, 12 Nov 2019 11:08:47 +0100 Subject: [PATCH 035/143] replace LambdaLR scheduler wrappers by function Custom schedulers are currently initiated by wrapping Pytorch's LambdaLR class and passing a method of the wrapping class to the __init__ function of LambdaLR. This approach is not appropriate for several reasons: 1. one does not need to define a class when it only defines a __init__() method; 2. instantiating the parent class by passing a method of the child class creates a cyclical reference which leads to memory leaks. See issues #1742 and #1134. In this commit we replace the wrapper classes with functions that instantiate `LambdaLR` with a custom learning rate function. We use a closure to specify the parameter of the latter. We also do a bit of renaming within the function to explicit the behaviour and removed docstrings that were subsequently not necessary. --- transformers/__init__.py | 4 +- transformers/optimization.py | 108 +++++++++--------------- transformers/tests/optimization_test.py | 29 ++++--- 3 files changed, 61 insertions(+), 80 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 53f3c39dc7..426f3bd3a2 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -97,8 +97,8 @@ if is_torch_available(): from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model # Optimization - from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, - WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) + from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, + get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) # TensorFlow diff --git a/transformers/optimization.py b/transformers/optimization.py index a48b5fea54..99e6cc75e4 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -23,89 +23,65 @@ from torch.optim.lr_scheduler import LambdaLR logger = logging.getLogger(__name__) -class ConstantLRSchedule(LambdaLR): - """ Constant learning rate schedule. + +def get_constant_schedule(optimizer, last_epoch=-1): + """ Create a schedule with a constant learning rate. """ - def __init__(self, optimizer, last_epoch=-1): - super(ConstantLRSchedule, self).__init__(optimizer, lambda _: 1.0, last_epoch=last_epoch) + return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch) -class WarmupConstantSchedule(LambdaLR): - """ Linear warmup and then constant. - Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. - Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. - Keeps multiplicative variable equal to 1. after warmup_steps. +def get_constant_schedule_with_warmup(optimizer, num_warmup_steps, last_epoch=-1): + """ Create a schedule with a constant learning rate preceded by a warmup + period during which the learning rate increases linearly between 0 and 1. """ - def __init__(self, optimizer, warmup_steps, last_epoch=-1): - self.warmup_steps = warmup_steps - super(WarmupConstantSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch) - - def lr_lambda(self, step): - if step < self.warmup_steps: - return float(step) / float(max(1.0, self.warmup_steps)) + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1.0, num_warmup_steps)) return 1. + return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch) -class WarmupLinearSchedule(LambdaLR): - """ Linear warmup and then linear decay. - Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. - Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. - Linearly decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps. + +def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1): + """ Create a schedule with a learning rate that decreases linearly after + linearly increasing during a warmup period. """ - def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1): - self.warmup_steps = warmup_steps - self.t_total = t_total - super(WarmupLinearSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch) + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))) - def lr_lambda(self, step): - if step < self.warmup_steps: - return float(step) / float(max(1, self.warmup_steps)) - return max(0.0, float(self.t_total - step) / float(max(1.0, self.t_total - self.warmup_steps))) + return LambdaLR(optimizer, lr_lambda, last_epoch) -class WarmupCosineSchedule(LambdaLR): - """ Linear warmup and then cosine decay. - Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. - Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. - Decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve. - If `cycles` (default=0.5) is different from default, then the multiplicative variable follows cosine function after warmup. +def get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=.5, last_epoch=-1): + """ Create a schedule with a learning rate that decreases following the + values of the cosine function between 0 and `pi * cycles` after a warmup + period during which it increases linearly between 0 and 1. """ - def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1): - self.warmup_steps = warmup_steps - self.t_total = t_total - self.cycles = cycles - super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch) + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + return max(0., 0.5 * (1. + math.cos(math.pi * float(num_cycles) * 2. * progress))) - def lr_lambda(self, step): - if step < self.warmup_steps: - return float(step) / float(max(1.0, self.warmup_steps)) - # progress after warmup - progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps)) - return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress))) + return LambdaLR(optimizer, lr_lambda, last_epoch) -class WarmupCosineWithHardRestartsSchedule(LambdaLR): - """ Linear warmup and then cosine cycles with hard restarts. - Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step. - Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps. - If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying - learning rate (with hard restarts). +def get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=1., last_epoch=-1): + """ Create a schedule with a learning rate that decreases following the + values of the cosine function with several hard restarts, after a warmup + period during which it increases linearly between 0 and 1. """ - def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1): - self.warmup_steps = warmup_steps - self.t_total = t_total - self.cycles = cycles - super(WarmupCosineWithHardRestartsSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch) - - def lr_lambda(self, step): - if step < self.warmup_steps: - return float(step) / float(max(1, self.warmup_steps)) - # progress after warmup - progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps)) - if progress >= 1.0: - return 0.0 - return max(0.0, 0.5 * (1. + math.cos(math.pi * ((float(self.cycles) * progress) % 1.0)))) + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + if progress >= 1.: + return 0. + return max(0., 0.5 * (1. + math.cos(math.pi * ((float(num_cycles) * progress) % 1.)))) + return LambdaLR(optimizer, lr_lambda, last_epoch) class AdamW(Optimizer): diff --git a/transformers/tests/optimization_test.py b/transformers/tests/optimization_test.py index 84dbaca52a..ab9afbfcf7 100644 --- a/transformers/tests/optimization_test.py +++ b/transformers/tests/optimization_test.py @@ -25,8 +25,12 @@ from transformers import is_torch_available if is_torch_available(): import torch - from transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, - WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) + from transformers import (AdamW, + get_constant_schedule, + get_constant_schedule_with_warmup, + get_cosine_schedule_with_warmup, + get_cosine_with_hard_restarts_schedule_with_warmup, + get_linear_schedule_with_warmup) else: pytestmark = pytest.mark.skip("Require Torch") @@ -87,59 +91,60 @@ class ScheduleInitTest(unittest.TestCase): self.assertAlmostEqual(a, b, delta=tol) def test_constant_scheduler(self): - scheduler = ConstantLRSchedule(self.optimizer) + scheduler = get_constant_schedule(self.optimizer) lrs = unwrap_schedule(scheduler, self.num_steps) expected_learning_rates = [10.] * self.num_steps self.assertEqual(len(lrs[0]), 1) self.assertListEqual([l[0] for l in lrs], expected_learning_rates) - scheduler = ConstantLRSchedule(self.optimizer) + scheduler = get_constant_schedule(self.optimizer) lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps) self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2]) def test_warmup_constant_scheduler(self): - scheduler = WarmupConstantSchedule(self.optimizer, warmup_steps=4) + scheduler = get_constant_schedule_with_warmup(self.optimizer, num_warmup_steps=4) lrs = unwrap_schedule(scheduler, self.num_steps) expected_learning_rates = [2.5, 5.0, 7.5, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0] self.assertEqual(len(lrs[0]), 1) self.assertListEqual([l[0] for l in lrs], expected_learning_rates) - scheduler = WarmupConstantSchedule(self.optimizer, warmup_steps=4) + scheduler = get_constant_schedule_with_warmup(self.optimizer, num_warmup_steps=4) lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps) self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2]) def test_warmup_linear_scheduler(self): - scheduler = WarmupLinearSchedule(self.optimizer, warmup_steps=2, t_total=10) + scheduler = get_linear_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10) lrs = unwrap_schedule(scheduler, self.num_steps) expected_learning_rates = [5.0, 10.0, 8.75, 7.5, 6.25, 5.0, 3.75, 2.5, 1.25, 0.0] self.assertEqual(len(lrs[0]), 1) self.assertListEqual([l[0] for l in lrs], expected_learning_rates) - scheduler = WarmupLinearSchedule(self.optimizer, warmup_steps=2, t_total=10) + scheduler = get_linear_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10) lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps) self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2]) def test_warmup_cosine_scheduler(self): - scheduler = WarmupCosineSchedule(self.optimizer, warmup_steps=2, t_total=10) + scheduler = get_cosine_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10) lrs = unwrap_schedule(scheduler, self.num_steps) expected_learning_rates = [5.0, 10.0, 9.61, 8.53, 6.91, 5.0, 3.08, 1.46, 0.38, 0.0] self.assertEqual(len(lrs[0]), 1) self.assertListAlmostEqual([l[0] for l in lrs], expected_learning_rates, tol=1e-2) - scheduler = WarmupCosineSchedule(self.optimizer, warmup_steps=2, t_total=10) + scheduler = get_cosine_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10) lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps) self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2]) def test_warmup_cosine_hard_restart_scheduler(self): - scheduler = WarmupCosineWithHardRestartsSchedule(self.optimizer, warmup_steps=2, cycles=2, t_total=10) + scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_cycles=2, num_training_steps=10) lrs = unwrap_schedule(scheduler, self.num_steps) expected_learning_rates = [5.0, 10.0, 8.53, 5.0, 1.46, 10.0, 8.53, 5.0, 1.46, 0.0] self.assertEqual(len(lrs[0]), 1) self.assertListAlmostEqual([l[0] for l in lrs], expected_learning_rates, tol=1e-2) - scheduler = WarmupCosineWithHardRestartsSchedule(self.optimizer, warmup_steps=2, cycles=2, t_total=10) + scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_cycles=2, num_training_steps=10) lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps) self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2]) + if __name__ == "__main__": unittest.main() From e18f786cd57c8cb84a09c6ce5e0d7de0dd8b106e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 14 Nov 2019 10:06:00 -0500 Subject: [PATCH 036/143] Quickstart example showcasing past --- docs/source/quickstart.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index ccba75e7c0..530aff8eb0 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -188,3 +188,35 @@ assert predicted_text == 'Who was Jim Henson? Jim Henson was a man' ``` Examples for each model class of each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [documentation](#documentation). + +#### Using the past + +GPT-2 as well as some other models (GPT, XLNet, Transfo-XL, CTRL) make use of a `past` or `mems` attribute which can be used to prevent re-computing the key/value pairs when using sequential decoding. It is useful when generating sequences as a big part of the attention mechanism benefits from previous computations. + +Here is a fully-working example using the `past` with `GPT2LMHeadModel` and argmax decoding (which should only be used as an example, as argmax decoding introduces a lot of repetition): + +```python +from transformers import GPT2LMHeadModel, GPT2Tokenizer +import torch + +tokenizer = GPT2Tokenizer.from_pretrained("gpt2") +model = GPT2LMHeadModel.from_pretrained('gpt2') + +generated = tokenizer.encode("The Manhattan bridge") +context = torch.tensor([generated]) +past = None + +for i in range(100): + print(i) + output, past = model(context, past=past) + token = torch.argmax(output[0, :]) + + generated += [token.tolist()] + context = token.unsqueeze(0) + +sequence = tokenizer.decode(generated) + +print(sequence) +``` + +The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`. \ No newline at end of file From a67e7478894c477822760ad7f9933a7d78aa27bd Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 14 Nov 2019 10:30:22 -0500 Subject: [PATCH 037/143] Reorganized max_len warning --- transformers/tokenization_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cd14cc4582..c5f469800b 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -671,10 +671,6 @@ class PreTrainedTokenizer(object): ids = [] for token in tokens: ids.append(self._convert_token_to_id_with_added_voc(token)) - if len(ids) > self.max_len: - logger.warning("Token indices sequence length is longer than the specified maximum sequence length " - "for this model ({} > {}). Running this sequence through the model will result in " - "indexing errors".format(len(ids), self.max_len)) return ids def _convert_token_to_id_with_added_voc(self, token): @@ -877,6 +873,11 @@ class PreTrainedTokenizer(object): encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] + if max_length is None and len(encoded_inputs["input_ids"]) > self.max_len: + logger.warning("Token indices sequence length is longer than the specified maximum sequence length " + "for this model ({} > {}). Running this sequence through the model will result in " + "indexing errors".format(len(ids), self.max_len)) + return encoded_inputs def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0): From d7929899daaca2d0b910dcef1181ea496f0b4909 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 14 Nov 2019 10:49:00 -0500 Subject: [PATCH 038/143] Specify checkpoint in saved file for run_lm_finetuning.py --- examples/run_lm_finetuning.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 2044cfe9e8..a143d55894 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -63,10 +63,10 @@ MODEL_CLASSES = { class TextDataset(Dataset): - def __init__(self, tokenizer, file_path='train', block_size=512): + def __init__(self, tokenizer, args, file_path='train', block_size=512): assert os.path.isfile(file_path) directory, filename = os.path.split(file_path) - cached_features_file = os.path.join(directory, 'cached_lm_' + str(block_size) + '_' + filename) + cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename) if os.path.exists(cached_features_file): logger.info("Loading features from cached file %s", cached_features_file) @@ -99,7 +99,7 @@ class TextDataset(Dataset): def load_and_cache_examples(args, tokenizer, evaluate=False): - dataset = TextDataset(tokenizer, file_path=args.eval_data_file if evaluate else args.train_data_file, block_size=args.block_size) + dataset = TextDataset(tokenizer, args, file_path=args.eval_data_file if evaluate else args.train_data_file, block_size=args.block_size) return dataset From 2276bf69b771763a2553eccd5d70c2b7331b1f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 14 Nov 2019 18:00:14 +0100 Subject: [PATCH 039/143] update the examples, docs and template --- README.md | 8 ++++---- docs/source/main_classes/optimizer_schedules.rst | 14 +++++--------- docs/source/migration.md | 8 ++++---- examples/contrib/run_openai_gpt.py | 4 ++-- examples/contrib/run_swag.py | 4 ++-- examples/distillation/distiller.py | 8 ++++---- examples/distillation/run_squad_w_distillation.py | 4 ++-- examples/run_glue.py | 4 ++-- examples/run_lm_finetuning.py | 4 ++-- examples/run_multiple_choice.py | 4 ++-- examples/run_ner.py | 4 ++-- examples/run_squad.py | 4 ++-- templates/adding_a_new_example_script/run_xxx.py | 4 ++-- 13 files changed, 35 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 40b08583b1..a22a3c83f5 100644 --- a/README.md +++ b/README.md @@ -521,12 +521,12 @@ Here is a conversion examples from `BertAdam` with a linear warmup and decay sch # Parameters: lr = 1e-3 max_grad_norm = 1.0 -num_total_steps = 1000 +num_training_steps = 1000 num_warmup_steps = 100 -warmup_proportion = float(num_warmup_steps) / float(num_total_steps) # 0.1 +warmup_proportion = float(num_warmup_steps) / float(num_training_steps) # 0.1 ### Previously BertAdam optimizer was instantiated like this: -optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps) +optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_training_steps) ### and used like this: for batch in train_data: loss = model(batch) @@ -535,7 +535,7 @@ for batch in train_data: ### In Transformers, optimizer and schedules are splitted and instantiated like this: optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False) # To reproduce BertAdam specific behavior set correct_bias=False -scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps) # PyTorch scheduler +scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps) # PyTorch scheduler ### and used like this: for batch in train_data: model.train() diff --git a/docs/source/main_classes/optimizer_schedules.rst b/docs/source/main_classes/optimizer_schedules.rst index ff0c9e6929..b30a2e0e2e 100644 --- a/docs/source/main_classes/optimizer_schedules.rst +++ b/docs/source/main_classes/optimizer_schedules.rst @@ -18,19 +18,17 @@ Schedules Learning Rate Schedules ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. autoclass:: transformers.ConstantLRSchedule - :members: +.. autofunction:: transformers.get_constant_schedule -.. autoclass:: transformers.WarmupConstantSchedule - :members: +.. autofunction:: transformers.get_constant_schedule_with_warmup .. image:: /imgs/warmup_constant_schedule.png :target: /imgs/warmup_constant_schedule.png :alt: -.. autoclass:: transformers.WarmupCosineSchedule +.. autofunction:: transformers.get_cosine_schedule_with_warmup :members: .. image:: /imgs/warmup_cosine_schedule.png @@ -38,8 +36,7 @@ Learning Rate Schedules :alt: -.. autoclass:: transformers.WarmupCosineWithHardRestartsSchedule - :members: +.. autofunction:: transformers.get_cosine_with_hard_restarts_schedule_with_warmup .. image:: /imgs/warmup_cosine_hard_restarts_schedule.png :target: /imgs/warmup_cosine_hard_restarts_schedule.png @@ -47,8 +44,7 @@ Learning Rate Schedules -.. autoclass:: transformers.WarmupLinearSchedule - :members: +.. autofunction:: transformers.get_linear_schedule_with_warmup .. image:: /imgs/warmup_linear_schedule.png :target: /imgs/warmup_linear_schedule.png diff --git a/docs/source/migration.md b/docs/source/migration.md index 553a79c82b..d04b66d5e4 100644 --- a/docs/source/migration.md +++ b/docs/source/migration.md @@ -84,12 +84,12 @@ Here is a conversion examples from `BertAdam` with a linear warmup and decay sch # Parameters: lr = 1e-3 max_grad_norm = 1.0 -num_total_steps = 1000 +num_training_steps = 1000 num_warmup_steps = 100 -warmup_proportion = float(num_warmup_steps) / float(num_total_steps) # 0.1 +warmup_proportion = float(num_warmup_steps) / float(num_training_steps) # 0.1 ### Previously BertAdam optimizer was instantiated like this: -optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps) +optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, num_training_steps=num_training_steps) ### and used like this: for batch in train_data: loss = model(batch) @@ -98,7 +98,7 @@ for batch in train_data: ### In Transformers, optimizer and schedules are splitted and instantiated like this: optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False) # To reproduce BertAdam specific behavior set correct_bias=False -scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps) # PyTorch scheduler +scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps) # PyTorch scheduler ### and used like this: for batch in train_data: loss = model(batch) diff --git a/examples/contrib/run_openai_gpt.py b/examples/contrib/run_openai_gpt.py index 7eb1b0be76..2d165a91e3 100644 --- a/examples/contrib/run_openai_gpt.py +++ b/examples/contrib/run_openai_gpt.py @@ -41,7 +41,7 @@ from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, from transformers import (OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer, AdamW, cached_path, WEIGHTS_NAME, CONFIG_NAME, - WarmupLinearSchedule) + get_linear_schedule_with_warmup) ROCSTORIES_URL = "https://s3.amazonaws.com/datasets.huggingface.co/ROCStories.tar.gz" @@ -211,7 +211,7 @@ def main(): {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.do_train: nb_tr_steps, tr_loss, exp_average_loss = 0, 0, None diff --git a/examples/contrib/run_swag.py b/examples/contrib/run_swag.py index 8494c5fad9..5de93db7fe 100644 --- a/examples/contrib/run_swag.py +++ b/examples/contrib/run_swag.py @@ -42,7 +42,7 @@ from tqdm import tqdm, trange from transformers import (WEIGHTS_NAME, BertConfig, BertForMultipleChoice, BertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup logger = logging.getLogger(__name__) @@ -322,7 +322,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py index d51bdae77f..0442072e84 100644 --- a/examples/distillation/distiller.py +++ b/examples/distillation/distiller.py @@ -35,7 +35,7 @@ try: except: from tensorboardX import SummaryWriter -from transformers import WarmupLinearSchedule +from transformers import get_linear_schedule_with_warmup from utils import logger from lm_seqs_dataset import LmSeqsDataset @@ -137,9 +137,9 @@ class Distiller: betas=(0.9, 0.98)) warmup_steps = math.ceil(num_train_optimization_steps * params.warmup_prop) - self.scheduler = WarmupLinearSchedule(self.optimizer, - warmup_steps=warmup_steps, - t_total=num_train_optimization_steps) + self.scheduler = get_linear_schedule_with_warmup(self.optimizer, + num_warmup_steps=warmup_steps, + num_training_steps=num_train_optimization_steps) if self.fp16: try: diff --git a/examples/distillation/run_squad_w_distillation.py b/examples/distillation/run_squad_w_distillation.py index 7c662df010..70b65dc1b8 100644 --- a/examples/distillation/run_squad_w_distillation.py +++ b/examples/distillation/run_squad_w_distillation.py @@ -46,7 +46,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from ..utils_squad import (read_squad_examples, convert_examples_to_features, RawResult, write_predictions, @@ -101,7 +101,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/examples/run_glue.py b/examples/run_glue.py index 1558a812c3..27048ad565 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -49,7 +49,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from transformers import glue_compute_metrics as compute_metrics from transformers import glue_output_modes as output_modes @@ -100,7 +100,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 2044cfe9e8..0085aee727 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -42,7 +42,7 @@ except: from tqdm import tqdm, trange -from transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule, +from transformers import (WEIGHTS_NAME, AdamW, get_linear_schedule_with_warmup, BertConfig, BertForMaskedLM, BertTokenizer, GPT2Config, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, @@ -185,7 +185,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py index 638bbe74f1..544014fb66 100644 --- a/examples/run_multiple_choice.py +++ b/examples/run_multiple_choice.py @@ -43,7 +43,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, RobertaConfig, RobertaForMultipleChoice, RobertaTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from utils_multiple_choice import (convert_examples_to_features, processors) @@ -101,7 +101,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/examples/run_ner.py b/examples/run_ner.py index b35d8298fe..0077080aec 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -33,7 +33,7 @@ from torch.utils.data.distributed import DistributedSampler from tqdm import tqdm, trange from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer @@ -80,7 +80,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id): {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/examples/run_squad.py b/examples/run_squad.py index d9dc2abfde..b954a8b8b9 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -45,7 +45,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from utils_squad import (read_squad_examples, convert_examples_to_features, RawResult, write_predictions, @@ -100,7 +100,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp diff --git a/templates/adding_a_new_example_script/run_xxx.py b/templates/adding_a_new_example_script/run_xxx.py index 489dcb19c7..77ce587a54 100644 --- a/templates/adding_a_new_example_script/run_xxx.py +++ b/templates/adding_a_new_example_script/run_xxx.py @@ -43,7 +43,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from utils_squad import (read_squad_examples, convert_examples_to_features, RawResult, write_predictions, @@ -98,7 +98,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp From 8f8d69716a1a61c5bc2d9cdcdb9147cfeb8441d1 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 14 Nov 2019 14:48:21 -0500 Subject: [PATCH 040/143] [CI][DOC] Don't rebuild if folder exists. --- .circleci/deploy.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 2bff0102ae..7eea0d0113 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -5,8 +5,12 @@ function deploy_doc(){ git checkout $1 if [ ! -z "$2" ] then - echo "Pushing version" $2 - make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 + if [ -d "$DIRECTORY" ]; then + echo "Directory" $2 "already exists" + else + echo "Pushing version" $2 + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 + fi else echo "Pushing master" make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir From be7f2aacce6d2e9e6a4a6a9e04b99ad0e660d703 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 14 Nov 2019 14:54:44 -0500 Subject: [PATCH 041/143] [CI][DOC] Don't rebuild if folder exists - Correct directory. --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 7eea0d0113..98151963d8 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -5,7 +5,7 @@ function deploy_doc(){ git checkout $1 if [ ! -z "$2" ] then - if [ -d "$DIRECTORY" ]; then + if [ -d "$dir/$2" ]; then echo "Directory" $2 "already exists" else echo "Pushing version" $2 From 05db5bc1afea196e548ae3214d3413c321fcfda1 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Thu, 14 Nov 2019 22:40:22 +0100 Subject: [PATCH 042/143] added small comparison between BERT, RoBERTa and DistilBERT --- examples/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/examples/README.md b/examples/README.md index 2b66b92f1a..abb4cb6e5a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -554,6 +554,16 @@ On the test dataset the following results could be achieved: 10/04/2019 00:42:42 - INFO - __main__ - recall = 0.8624150210424085 ``` +### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) + +Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) with the same hyperparameters as specified in the [example documentation](https://huggingface.co/transformers/examples.html#named-entity-recognition) (one run): + +| Model | F-Score Dev | F-Score Test +| --------------------------------- | ------- | -------- +| `bert-large-cased` | 95.59 | 91.70 +| `roberta-large` | 95.96 | 91.87 +| `distilbert-base-uncased` | 94.34 | 90.32 + ## Abstractive summarization Based on the script From 14b3aa3b3c300eb1fcc4f3a0c046c87bdabe0afd Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Fri, 8 Nov 2019 16:47:21 -0800 Subject: [PATCH 043/143] Add tokenization_camembert.py --- transformers/tokenization_camembert.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 transformers/tokenization_camembert.py diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py new file mode 100644 index 0000000000..9facf7d911 --- /dev/null +++ b/transformers/tokenization_camembert.py @@ -0,0 +1,120 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization classes for Camembert model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer + + +class CamembertTokenizer(PreTrainedTokenizer): + """ + Adapted from RobertaTokenizer and XLNetTokenizer + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + vocab_files_names = {'vocab_file': None} + + def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", + cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): + super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, + mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(str(vocab_file)) + # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual + # sentencepiece vocabulary (this is the case for and + self.fairseq_tokens_to_ids = {'NOTUSED': 0, '': 1, 'NOTUSED': 2, '': 3} + self.fairseq_offset = len(self.fairseq_tokens_to_ids) + self.fairseq_tokens_to_ids[''] = len(self.sp_model) + len(self.fairseq_tokens_to_ids) + self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()} + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] + + @property + def vocab_size(self): + return self.fairseq_offset + len(self.sp_model) + + def _tokenize(self, text): + return self.sp_model.EncodeAsPieces(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + if token in self.fairseq_tokens_to_ids: + return self.fairseq_tokens_to_ids[token] + return self.fairseq_offset + self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + if index in self.fairseq_ids_to_tokens: + return self.fairseq_ids_to_tokens[index] + return self.sp_model.IdToPiece(index - self.fairseq_offset) From 6e72fd094c98901cc90d146d3fe3cd5a0e879911 Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Fri, 8 Nov 2019 17:09:48 -0800 Subject: [PATCH 044/143] Add demo_camembert.py --- examples/demo_camembert.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 examples/demo_camembert.py diff --git a/examples/demo_camembert.py b/examples/demo_camembert.py new file mode 100644 index 0000000000..df28f4f267 --- /dev/null +++ b/examples/demo_camembert.py @@ -0,0 +1,59 @@ +from pathlib import Path +import tarfile +import urllib.request + +import torch + +from transformers.tokenization_camembert import CamembertTokenizer +from transformers.modeling_roberta import RobertaForMaskedLM + + +def fill_mask(masked_input, model, tokenizer, topk=5): + # Adapted from https://github.com/pytorch/fairseq/blob/master/fairseq/models/roberta/hub_interface.py + assert masked_input.count('') == 1 + input_ids = torch.tensor(tokenizer.encode(masked_input, add_special_tokens=True)).unsqueeze(0) # Batch size 1 + logits = model(input_ids)[0] # The last hidden-state is the first element of the output tuple + masked_index = (input_ids.squeeze() == tokenizer.mask_token_id).nonzero().item() + logits = logits[0, masked_index, :] + prob = logits.softmax(dim=0) + values, indices = prob.topk(k=topk, dim=0) + topk_predicted_token_bpe = ' '.join([tokenizer.convert_ids_to_tokens(indices[i].item()) + for i in range(len(indices))]) + masked_token = tokenizer.mask_token + topk_filled_outputs = [] + for index, predicted_token_bpe in enumerate(topk_predicted_token_bpe.split(' ')): + predicted_token = predicted_token_bpe.replace('\u2581', ' ') + if " {0}".format(masked_token) in masked_input: + topk_filled_outputs.append(( + masked_input.replace( + ' {0}'.format(masked_token), predicted_token + ), + values[index].item(), + predicted_token, + )) + else: + topk_filled_outputs.append(( + masked_input.replace(masked_token, predicted_token), + values[index].item(), + predicted_token, + )) + return topk_filled_outputs + + +model_path = Path('camembert.v0.pytorch') +if not model_path.exists(): + compressed_path = model_path.with_suffix('.tar.gz') + url = 'http://dl.fbaipublicfiles.com/camembert/camembert.v0.pytorch.tar.gz' + print('Downloading model...') + urllib.request.urlretrieve(url, compressed_path) + print('Extracting model...') + with tarfile.open(compressed_path) as f: + f.extractall(model_path.parent) + assert model_path.exists() +tokenizer_path = model_path / 'sentencepiece.bpe.model' +tokenizer = CamembertTokenizer.from_pretrained(tokenizer_path) +model = RobertaForMaskedLM.from_pretrained(model_path) +model.eval() + +masked_input = "Le camembert est :)" +print(fill_mask(masked_input, model, tokenizer, topk=3)) From e44b939e7198fac5ce4085a1bda8e17b3934e67f Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:11:00 -0800 Subject: [PATCH 045/143] Add configuration_camembert.py and modeling_camembert.py --- transformers/configuration_camembert.py | 33 +++ transformers/modeling_camembert.py | 257 ++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 transformers/configuration_camembert.py create mode 100644 transformers/modeling_camembert.py diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py new file mode 100644 index 0000000000..07ebd6e82e --- /dev/null +++ b/transformers/configuration_camembert.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CamemBERT configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .configuration_roberta import RobertaConfig + +logger = logging.getLogger(__name__) + +CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-config.json", +} + + +class CamembertConfig(RobertaConfig): + pretrained_config_archive_map = CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py new file mode 100644 index 0000000000..982c349531 --- /dev/null +++ b/transformers/modeling_camembert.py @@ -0,0 +1,257 @@ +# coding=utf-8 +# Copyright 2019 Inria, Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch CamemBERT model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice +from .configuration_camembert import CamembertConfig +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-pytorch_model.bin", +} + + +CAMEMBERT_START_DOCSTRING = r""" The CamemBERT model was proposed in + `CamemBERT: a Tasty French Language Model`_ + by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suárez, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah, and Benoît Sagot. It is based on Facebook's RoBERTa model released in 2019. + + It is a model trained on 138GB of French text. + + This implementation is the same RoBERTa. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`CamemBERT: a Tasty French Language Model`: + https://arxiv.org/abs/1911.03894 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.CamembertConfig`): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +CAMEMBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, CamemBERT input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: Is this Jacksonville ? No it is not . `` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + Fully encoded sequences or sequence pairs can be obtained using the CamembertTokenizer.encode function with + the ``add_special_tokens`` parameter set to ``True``. + + CamemBERT is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional segment token indices to indicate first and second portions of the inputs. + This embedding matrice is not trained (not pretrained during CamemBERT pretraining), you will have to train it + during finetuning. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1[``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.", + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertModel(RobertaModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + eo match pre-training, CamemBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertModel.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" + + +@add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForMaskedLM(RobertaForMaskedLM): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForMaskedLM.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" + + +@add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer + on top of the pooled output) e.g. for GLUE tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForSequenceClassification(RobertaForSequenceClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForSequenceClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" + + +@add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForMultipleChoice(RobertaForMultipleChoice): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForMultipleChoice.from_pretrained('camembert-base') + choices = ["J'aime le camembert !", "Je deteste le camembert !"] + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" From fb6c70a91d3183742ce0a6d97add68103253ca3a Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:11:49 -0800 Subject: [PATCH 046/143] Update tokenization_camembert.py with urls --- transformers/tokenization_camembert.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 9facf7d911..0a6e751351 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -11,7 +11,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License """ Tokenization classes for Camembert model.""" from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -20,6 +20,19 @@ import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-sentencepiece.bpe.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'camembert-base': None, +} + class CamembertTokenizer(PreTrainedTokenizer): """ Adapted from RobertaTokenizer and XLNetTokenizer @@ -27,7 +40,9 @@ class CamembertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ - vocab_files_names = {'vocab_file': None} + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): From f12e4d8da783a535cd8978656f0a3ca108a2e2ed Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:14:50 -0800 Subject: [PATCH 047/143] Move demo_camembert.py to examples/contrib --- examples/{ => contrib}/demo_camembert.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{ => contrib}/demo_camembert.py (100%) diff --git a/examples/demo_camembert.py b/examples/contrib/demo_camembert.py similarity index 100% rename from examples/demo_camembert.py rename to examples/contrib/demo_camembert.py From 3e20c2e871db82f81c3b2b814265a481be15273c Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:16:24 -0800 Subject: [PATCH 048/143] Update demo_camembert.py with new classes --- examples/contrib/demo_camembert.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/examples/contrib/demo_camembert.py b/examples/contrib/demo_camembert.py index df28f4f267..28144d5167 100644 --- a/examples/contrib/demo_camembert.py +++ b/examples/contrib/demo_camembert.py @@ -5,7 +5,7 @@ import urllib.request import torch from transformers.tokenization_camembert import CamembertTokenizer -from transformers.modeling_roberta import RobertaForMaskedLM +from transformers.modeling_camembert import CamembertForMaskedLM def fill_mask(masked_input, model, tokenizer, topk=5): @@ -40,19 +40,8 @@ def fill_mask(masked_input, model, tokenizer, topk=5): return topk_filled_outputs -model_path = Path('camembert.v0.pytorch') -if not model_path.exists(): - compressed_path = model_path.with_suffix('.tar.gz') - url = 'http://dl.fbaipublicfiles.com/camembert/camembert.v0.pytorch.tar.gz' - print('Downloading model...') - urllib.request.urlretrieve(url, compressed_path) - print('Extracting model...') - with tarfile.open(compressed_path) as f: - f.extractall(model_path.parent) - assert model_path.exists() -tokenizer_path = model_path / 'sentencepiece.bpe.model' -tokenizer = CamembertTokenizer.from_pretrained(tokenizer_path) -model = RobertaForMaskedLM.from_pretrained(model_path) +tokenizer = CamembertTokenizer.from_pretrained('camembert-base') +model = CamembertForMaskedLM.from_pretrained('camembert-base') model.eval() masked_input = "Le camembert est :)" From 694d4fcbb61f15b66781219954112791248d832e Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:28:37 -0800 Subject: [PATCH 049/143] Add CamemBERT classes to __init__.py --- transformers/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index dd9e18a050..cdf0669b39 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_camembert import CamembertTokenizer # Configurations from .configuration_utils import PretrainedConfig @@ -56,6 +57,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling if is_torch_available(): @@ -96,6 +98,9 @@ if is_torch_available(): DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DistilBertForTokenClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_camembert import (CamembertForMaskedLM, CamembertModel, + CamembertForSequenceClassification, CamembertForMultipleChoice, + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model # Optimization From 035fea53157b43d311b1e1164395398c4ec0dda5 Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:41:41 -0800 Subject: [PATCH 050/143] Add CamemBERT to auto files and docs --- docs/source/pretrained_models.rst | 6 +++++- transformers/configuration_auto.py | 7 ++++++- transformers/tokenization_auto.py | 9 +++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index edb47e7f1c..b0a578fd80 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -155,5 +155,9 @@ Here is the full list of the currently provided pretrained models together with | CTRL | ``ctrl`` | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters | | | | | Salesforce's Large-sized CTRL English model | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| CamemBERT | ``camembert-base`` | | 12-layer, 768-hidden, 12-heads, 110M parameters | +| | | | CamemBERT using the BERT-base architecture | +| | | (see `details `__) | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -.. `__ \ No newline at end of file +.. `__ diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a670c..2906136139 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -27,6 +27,7 @@ from .configuration_xlm import XLMConfig from .configuration_roberta import RobertaConfig from .configuration_distilbert import DistilBertConfig from .configuration_ctrl import CTRLConfig +from .configuration_camembert import CamembertConfig logger = logging.getLogger(__name__) @@ -50,6 +51,7 @@ class AutoConfig(object): - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - contains `roberta`: RobertaConfig (RoBERTa model) + - contains `camembert`: CamembertConfig (CamemBERT model) - contains `ctrl` : CTRLConfig (CTRL model) This class cannot be instantiated using `__init__()` (throw an error). """ @@ -72,6 +74,7 @@ class AutoConfig(object): - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - contains `roberta`: RobertaConfig (RoBERTa model) + - contains `camembert`: CamembertConfig (CamemBERT model) - contains `ctrl` : CTRLConfig (CTRL model) Params: pretrained_model_name_or_path: either: @@ -116,6 +119,8 @@ class AutoConfig(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -134,4 +139,4 @@ class AutoConfig(object): return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path)) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de17f..4510159905 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -27,6 +27,7 @@ from .tokenization_xlnet import XLNetTokenizer from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_camembert import CamembertTokenizer logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) @@ -64,8 +66,9 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - - contains `roberta`: RobertaTokenizer (XLM model) + - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) @@ -103,6 +106,8 @@ class AutoTokenizer(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -121,4 +126,4 @@ class AutoTokenizer(object): return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path)) From 26858f27cb352b6bb1cda2b090413d1d2206a9ee Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 Nov 2019 23:23:31 -0500 Subject: [PATCH 051/143] [camembert] Upload to s3 + rename script --- examples/contrib/{demo_camembert.py => run_camembert.py} | 0 transformers/configuration_camembert.py | 2 +- transformers/modeling_camembert.py | 2 +- transformers/tokenization_camembert.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename examples/contrib/{demo_camembert.py => run_camembert.py} (100%) diff --git a/examples/contrib/demo_camembert.py b/examples/contrib/run_camembert.py similarity index 100% rename from examples/contrib/demo_camembert.py rename to examples/contrib/run_camembert.py diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py index 07ebd6e82e..3ff64454e5 100644 --- a/transformers/configuration_camembert.py +++ b/transformers/configuration_camembert.py @@ -25,7 +25,7 @@ from .configuration_roberta import RobertaConfig logger = logging.getLogger(__name__) CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-config.json", + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json", } diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index 982c349531..a0f5933bc2 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -27,7 +27,7 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-pytorch_model.bin", + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin", } diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 0a6e751351..de587ac863 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -25,7 +25,7 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-sentencepiece.bpe.model", + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model", } } From f9abf73e319d99ed74f46671e98b00b9328cd245 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 Nov 2019 23:40:20 -0500 Subject: [PATCH 052/143] [camembert] realign w/ recent changes --- transformers/modeling_camembert.py | 10 +++++----- transformers/tokenization_camembert.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index a0f5933bc2..05538926e2 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -37,7 +37,7 @@ CAMEMBERT_START_DOCSTRING = r""" The CamemBERT model was proposed in It is a model trained on 138GB of French text. - This implementation is the same RoBERTa. + This implementation is the same as RoBERTa. This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. @@ -94,6 +94,10 @@ CAMEMBERT_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.", @@ -143,7 +147,6 @@ class CamembertModel(RobertaModel): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" @add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """, @@ -180,7 +183,6 @@ class CamembertForMaskedLM(RobertaForMaskedLM): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" @add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer @@ -219,7 +221,6 @@ class CamembertForSequenceClassification(RobertaForSequenceClassification): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" @add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of @@ -254,4 +255,3 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index de587ac863..ae1b322941 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -87,7 +87,7 @@ class CamembertTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: if token_ids_1 is not None: From 0477b307c7501ea76e01b03cb387a2312db752b3 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 Nov 2019 23:54:11 -0500 Subject: [PATCH 053/143] [camembert] tokenizer: use additional_special_tokens --- transformers/tokenization_camembert.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index ae1b322941..41d3d74cff 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -45,10 +45,12 @@ class CamembertTokenizer(PreTrainedTokenizer): max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", - cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): + cls_token="", unk_token="", pad_token='', mask_token='', + additional_special_tokens=['NOTUSED', 'NOTUSED'], **kwargs): super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, - mask_token=mask_token, **kwargs) + mask_token=mask_token, additional_special_tokens=additional_special_tokens, + **kwargs) self.max_len_single_sentence = self.max_len - 2 # take into account special tokens self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens self.sp_model = spm.SentencePieceProcessor() From d32ce2c8df7053c19061b709465cdcc765e45a15 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 14:14:19 +0100 Subject: [PATCH 054/143] camembert: add wrapper for CamembertForTokenClassification --- transformers/modeling_camembert.py | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index 05538926e2..f302346f2d 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function, import logging -from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification from .configuration_camembert import CamembertConfig from .file_utils import add_start_docstrings @@ -255,3 +255,39 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForTokenClassification(RobertaForTokenClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForTokenClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !", add_special_tokens=True)).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP From 33753d9139307d9635db0309b6ddb9c53192c60a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 14:14:54 +0100 Subject: [PATCH 055/143] module: import CamembertForTokenClassification --- transformers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index cdf0669b39..5c7b0a6197 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -100,6 +100,7 @@ if is_torch_available(): DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_camembert import (CamembertForMaskedLM, CamembertModel, CamembertForSequenceClassification, CamembertForMultipleChoice, + CamembertForTokenClassification, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model From 44455eb5b61b2fc4a0bdbea6775d328c45a597e5 Mon Sep 17 00:00:00 2001 From: Sebastian Stabinger Date: Mon, 18 Nov 2019 10:07:05 +0100 Subject: [PATCH 056/143] Adds CamemBERT to Model architectures list --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 40d6acb76e..d1b37dbd48 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,8 @@ At some point in the future, you'll be able to seamlessly move from pre-training 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. -10. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. +10. **[CamemBERT](https://camembert-model.fr)** (from Facebook, Inria, Sorbonne) French language model based on RoBERTa released together with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Martin, Muller, and Suarez et al. +11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). From 3916b334a86484af8442d1cfdb2f15695feae581 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 18 Nov 2019 09:29:11 -0500 Subject: [PATCH 057/143] [camembert] Acknowledge the full author list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d1b37dbd48..a49e8086d1 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. -10. **[CamemBERT](https://camembert-model.fr)** (from Facebook, Inria, Sorbonne) French language model based on RoBERTa released together with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Martin, Muller, and Suarez et al. +10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). From 0b3d45eb64607158977f546d57f90eae268c7836 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 15:49:44 +0100 Subject: [PATCH 058/143] camembert: add implementation for save_vocabulary method --- transformers/tokenization_camembert.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 41d3d74cff..bf2a6fe993 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -16,9 +16,14 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import logging +import os +from shutil import copyfile + import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +logger = logging.getLogger(__name__) VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} @@ -55,6 +60,7 @@ class CamembertTokenizer(PreTrainedTokenizer): self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(str(vocab_file)) + self.vocab_file = vocab_file # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual # sentencepiece vocabulary (this is the case for and self.fairseq_tokens_to_ids = {'NOTUSED': 0, '': 1, 'NOTUSED': 2, '': 3} @@ -135,3 +141,17 @@ class CamembertTokenizer(PreTrainedTokenizer): if index in self.fairseq_ids_to_tokens: return self.fairseq_ids_to_tokens[index] return self.sp_model.IdToPiece(index - self.fairseq_offset) + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From 56c84863a1a20dfb82b928c5c9f77c21d9def8c7 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 15:50:16 +0100 Subject: [PATCH 059/143] camembert: add support for CamemBERT in run_ner example --- examples/run_ner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 4359e587ae..127d63a6cd 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -37,6 +37,7 @@ from transformers import AdamW, get_linear_schedule_with_warmup from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer +from transformers import CamembertConfig, CamembertForTokenClassification, CamembertTokenizer logger = logging.getLogger(__name__) @@ -47,7 +48,8 @@ ALL_MODELS = sum( MODEL_CLASSES = { "bert": (BertConfig, BertForTokenClassification, BertTokenizer), "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer), - "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer) + "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer), + "camembert": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer), } From f3386d938348628c91457fc7d8650c223317a053 Mon Sep 17 00:00:00 2001 From: Kazutoshi Shinoda Date: Sun, 17 Nov 2019 18:08:51 +0900 Subject: [PATCH 060/143] typo "deay" -> "decay" --- examples/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index d7fdc32ae7..69088d73c3 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -402,7 +402,7 @@ def main(): parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, From 454455c695ff38df1ed3670a43677fdd1abcedf3 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 20 Nov 2019 09:42:48 -0500 Subject: [PATCH 061/143] fix #1879 --- examples/utils_squad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/utils_squad.py b/examples/utils_squad.py index c04dacf6d3..4f1c581588 100644 --- a/examples/utils_squad.py +++ b/examples/utils_squad.py @@ -240,6 +240,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length, # The -3 accounts for [CLS], [SEP] and [SEP] max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + assert max_tokens_for_doc > 0 # We can have documents that are longer than the maximum sequence length. # To deal with this we do a sliding window approach, where we take chunks From e70cdf083ddb8bfe298d43e6d70d698a3a2f56d3 Mon Sep 17 00:00:00 2001 From: Jin Young Sohn Date: Wed, 20 Nov 2019 22:33:26 +0000 Subject: [PATCH 062/143] Cleanup TPU bits from run_glue.py TPU runner is currently implemented in: https://github.com/pytorch-tpu/transformers/blob/tpu/examples/run_glue_tpu.py. We plan to upstream this directly into `huggingface/transformers` (either `master` or `tpu`) branch once it's been more thoroughly tested. --- examples/run_glue.py | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 19316cb0ec..527e440075 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -158,7 +158,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -189,11 +189,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -397,15 +392,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -439,23 +425,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -509,7 +478,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) From 6f70bb8c69cdbe1207e4909ccface32c8f51297b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 20 Nov 2019 18:01:03 +0100 Subject: [PATCH 063/143] add instructions to run the examples --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index a49e8086d1..9cc783fd71 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,18 @@ When TensorFlow 2.0 and/or PyTorch has been installed, you can install from sour pip install [--editable] . ``` +### Run the examples + +Examples are included in the repository but are not shipped with the library. +Therefore, in order to run the examples you will first need to clone the +repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps: + +```bash +git clone git@github.com:huggingface/transformers +cd transformers +pip install . +``` + ### Tests A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). @@ -253,6 +265,11 @@ print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sen ## Quick tour of the fine-tuning/usage scripts +**Important** +Before running the fine-tuning scripts, please read the +[instructions](#run-the-examples) on how to +setup your environment to run the examples. + The library comprises several example scripts with SOTA performances for NLU and NLG tasks: - `run_glue.py`: an example fine-tuning Bert, XLNet and XLM on nine different GLUE tasks (*sequence-level classification*) From 26db31e0c09a8b5e1ca7a61c454b159eab9d86be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 20 Nov 2019 18:13:38 +0100 Subject: [PATCH 064/143] update the documentation --- examples/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/README.md b/examples/README.md index abb4cb6e5a..622fa07f8f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,6 +3,15 @@ In this section a few examples are put together. All of these examples work for several models, making use of the very similar API between the different models. +**Important** +To use the examples, execute the following steps in a new virtual environment: + +```bash +git clone git@github.com:huggingface/transformers +cd transformers +pip install . +``` + | Section | Description | |----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| | [TensorFlow 2.0 models on GLUE](#TensorFlow-2.0-Bert-models-on-GLUE) | Examples running BERT TensorFlow 2.0 model on the GLUE tasks. From 041a901f324eea7e7ee04b0f7a563c7ed5c8a03a Mon Sep 17 00:00:00 2001 From: Nikolay Korolev Date: Fri, 22 Nov 2019 22:56:43 +0300 Subject: [PATCH 065/143] Fix typo in documentation. toto -> to --- transformers/tokenization_gpt2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 4bec515903..a4798b0129 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -184,7 +184,7 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ Tokenize a string. Args: - add_prefix_space (boolean, default False): - Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers. + Begin the sentence with at least one space to get invariance to word order in GPT-2 (and RoBERTa) tokenizers. """ if add_prefix_space: text = ' ' + text From 176cd1ce1b337134425b426207fbe155099c18b4 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 23 Nov 2019 11:18:54 -0500 Subject: [PATCH 066/143] [doc] homogenize instructions slightly --- README.md | 5 ++--- examples/README.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9cc783fd71..52e74a6f80 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,12 @@ pip install [--editable] . ### Run the examples Examples are included in the repository but are not shipped with the library. -Therefore, in order to run the examples you will first need to clone the -repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps: +Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: ```bash git clone git@github.com:huggingface/transformers cd transformers -pip install . +pip install [--editable] . ``` ### Tests diff --git a/examples/README.md b/examples/README.md index 622fa07f8f..0c57990ea7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,12 +4,12 @@ In this section a few examples are put together. All of these examples work for similar API between the different models. **Important** -To use the examples, execute the following steps in a new virtual environment: +To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: ```bash git clone git@github.com:huggingface/transformers cd transformers -pip install . +pip install [--editable] . ``` | Section | Description | From afaa33585109550f9ecaaee4e47f187aaaefedd0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 23 Nov 2019 11:34:45 -0500 Subject: [PATCH 067/143] [doc] Fix assets urls --- docs/source/_static/js/custom.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index 2c7836fd20..ec804b3704 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -1,5 +1,5 @@ function addIcon() { - const huggingFaceLogo = "https://huggingface.co/assets/transformers-docs/huggingface_logo.svg"; + const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg"; const image = document.createElement("img"); image.setAttribute("src", huggingFaceLogo); @@ -24,10 +24,10 @@ function addCustomFooter() { social.classList.add("footer__Social"); const imageDetails = [ - { link: "https://huggingface.co", imageLink: "https://huggingface.co/assets/transformers-docs/website.svg" }, - { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/twitter.svg" }, - { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/github.svg" }, - { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/assets/transformers-docs/linkedin.svg" } + { link: "https://huggingface.co", imageLink: "https://huggingface.co/landing/assets/transformers-docs/website.svg" }, + { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/twitter.svg" }, + { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/github.svg" }, + { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/landing/assets/transformers-docs/linkedin.svg" } ]; imageDetails.forEach(imageLinks => { From 7485caefb09cb7f4c4b720b40ec69fed345a6b1c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 25 Nov 2019 09:33:39 -0500 Subject: [PATCH 068/143] fix #1894 --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 52a1b75a65..aded521c1d 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -68,7 +68,7 @@ class TextDataset(Dataset): directory, filename = os.path.split(file_path) cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename) - if os.path.exists(cached_features_file): + if os.path.exists(cached_features_file) and not args.overwrite_cache: logger.info("Loading features from cached file %s", cached_features_file) with open(cached_features_file, 'rb') as handle: self.examples = pickle.load(handle) From 99f750d64e78d20fa5213ea11235b6a1b084481e Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 10:35:07 +0100 Subject: [PATCH 069/143] add Camembert models to modeling_auto --- transformers/modeling_auto.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..a5828148ad 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -27,6 +27,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice from .modeling_utils import PreTrainedModel, SequenceSummary @@ -48,6 +49,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -71,6 +73,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -138,6 +141,8 @@ class AutoModel(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -172,6 +177,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -198,6 +204,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -264,6 +271,8 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + if 'camembert' in pretrained_model_name_or_path: + return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -298,6 +307,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -320,6 +330,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -383,6 +394,8 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + if 'camembert' in pretrained_model_name_or_path: + return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: From c8eb8157b86291413fe8096217f4defc168aa73f Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 11:01:20 +0100 Subject: [PATCH 070/143] fix docstrings --- transformers/modeling_auto.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index a5828148ad..ce36f6dc4a 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -49,7 +49,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -73,7 +73,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -177,7 +177,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -204,7 +204,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -307,7 +307,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -330,7 +330,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) From fa963ecc59a1dea59c2d0e952b2c4483e1828176 Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 11:03:12 +0100 Subject: [PATCH 071/143] =?UTF-8?q?if=E2=86=92elif?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- transformers/modeling_auto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index ce36f6dc4a..5866420001 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -271,7 +271,7 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - if 'camembert' in pretrained_model_name_or_path: + elif 'camembert' in pretrained_model_name_or_path: return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) @@ -394,7 +394,7 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - if 'camembert' in pretrained_model_name_or_path: + elif 'camembert' in pretrained_model_name_or_path: return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) From 07bf43074f39379d8c7f6afcca105e69685f7531 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Thu, 21 Nov 2019 20:52:06 -0600 Subject: [PATCH 072/143] Fix GPT2 docstring --- transformers/tokenization_gpt2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index a4798b0129..5fda709448 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -107,10 +107,10 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ GPT-2 BPE tokenizer. Peculiarities: - Byte-level Byte-Pair-Encoding - - Requires a space to start the input string => the encoding methods should be called with the + - Requires a space to start the input string => the encoding and tokenize methods should be called with the ``add_prefix_space`` flag set to ``True``. - Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve - the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"` + Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve + the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(" Hello")) = "Hello"` """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP From aa92a184d2b92faadec975139ad55e2ae749362c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0brahim=20Ethem=20Demirci?= Date: Sat, 16 Nov 2019 16:49:37 +0300 Subject: [PATCH 073/143] resize model when special tokenizer present --- examples/run_lm_finetuning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index aded521c1d..c33aa94a32 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -215,6 +215,7 @@ def train(args, train_dataset, model, tokenizer): global_step = 0 tr_loss, logging_loss = 0.0, 0.0 + model.resize_token_embeddings(len(tokenizer)) model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) From 5d3b8daad2cc6287d30f03f8a96d0a1f7bc8d0dc Mon Sep 17 00:00:00 2001 From: manansanghi <52307004+manansanghi@users.noreply.github.com> Date: Fri, 22 Nov 2019 10:31:54 -0800 Subject: [PATCH 074/143] Minor bug fixes on run_ner.py --- examples/run_ner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 127d63a6cd..1ab1236d94 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -127,7 +127,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id): "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -217,7 +217,7 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix="" "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] From 8e5d84fcc1a645d3c13b8a2f64fa995637440dad Mon Sep 17 00:00:00 2001 From: v_sboliu Date: Tue, 26 Nov 2019 12:04:31 +0800 Subject: [PATCH 075/143] Fixed typo --- transformers/modeling_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 7c2c6f4602..81d92d8f1b 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -278,7 +278,7 @@ class BertAttention(nn.Module): if len(heads) == 0: return mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + heads = set(heads) - self.pruned_heads # Convert to set and remove already pruned heads for head in heads: # Compute how many pruned heads are before the head and move the index accordingly head = head - sum(1 if h < head else 0 for h in self.pruned_heads) From c0c2088333e2e8ce2b24d0c7f4bf071dcccbd7ea Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 19:57:38 +0000 Subject: [PATCH 076/143] ALBERT model --- transformers/configuration_albert.py | 72 ++++ ...lbert_original_tf_checkpoint_to_pytorch.py | 44 +++ transformers/modeling_albert.py | 331 ++++++++++++++++++ 3 files changed, 447 insertions(+) create mode 100644 transformers/configuration_albert.py create mode 100644 transformers/convert_albert_original_tf_checkpoint_to_pytorch.py create mode 100644 transformers/modeling_albert.py diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py new file mode 100644 index 0000000000..c86c9565cb --- /dev/null +++ b/transformers/configuration_albert.py @@ -0,0 +1,72 @@ +from .configuration_utils import PretrainedConfig + +class AlbertConfig(PretrainedConfig): + """Configuration for `AlbertModel`. + + The default settings match the configuration of model `albert_xxlarge`. + """ + + def __init__(self, + vocab_size_or_config_json_file, + embedding_size=128, + hidden_size=4096, + num_hidden_layers=12, + num_hidden_groups=1, + num_attention_heads=64, + intermediate_size=16384, + inner_group_num=1, + down_scale_factor=1, + hidden_act="gelu", + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, **kwargs): + """Constructs AlbertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `AlbertModel`. + embedding_size: size of voc embeddings. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_hidden_groups: Number of group for the hidden layers, parameters in + the same group are shared. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + inner_group_num: int, number of inner repetition of attention and ffn. + down_scale_factor: float, the scale to apply + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `AlbertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + super(AlbertConfig, self).__init__(**kwargs) + + self.vocab_size = vocab_size_or_config_json_file + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_hidden_groups = num_hidden_groups + self.num_attention_heads = num_attention_heads + self.inner_group_num = inner_group_num + self.down_scale_factor = down_scale_factor + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps \ No newline at end of file diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py new file mode 100644 index 0000000000..04877d41b9 --- /dev/null +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,44 @@ + + + +from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert + + + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + # Initialise PyTorch model + config = BertConfig.from_json_file(bert_config_file) + print("Building PyTorch model from configuration: {}".format(str(config))) + model = BertForPreTraining(config) + + # Load weights from tf checkpoint + load_tf_weights_in_bert(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--albert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained BERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, + args.bert_config_file, + args.pytorch_dump_path) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py new file mode 100644 index 0000000000..b006cbe8fd --- /dev/null +++ b/transformers/modeling_albert.py @@ -0,0 +1,331 @@ + +import os +import math +import logging +import torch +import torch.nn as nn +from transformers.configuration_albert import AlbertConfig +logger = logging.getLogger(__name__) + +def load_tf_weights_in_albert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model.""" + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + print(model) + + for name, array in zip(names, arrays): + og = name + name = name.replace("transformer/group_0/inner_group_0", "transformer") + name = name.replace("LayerNorm", "layer_norm") + name = name.replace("ffn_1", "ffn") + name = name.replace("ffn/intermediate/output", "ffn_output") + name = name.replace("attention_1", "attention") + name = name.replace("cls/predictions/transform", "predictions") + name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm") + name = name.split('/') + + print(name) + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + print("transposed") + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + print("Initialize PyTorch weight {} from {}".format(name, og)) + pointer.data = torch.from_numpy(array) + + return model + + +class AlbertEmbeddings(nn.Module): + """ + Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(AlbertEmbeddings, self).__init__() + + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) + self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + word_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = word_embeddings + position_embeddings + token_type_embeddings + embeddings = self.layer_norm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + + def get_word_embeddings_table(self): + return self.word_embeddings + + +class AlbertModel(nn.Module): + def __init__(self, config): + super(AlbertModel, self).__init__() + + self.config = config + self.embeddings = AlbertEmbeddings(config) + self.encoder = AlbertEncoder(config) + self.pooler = nn.Linear(config.hidden_size, config.hidden_size) + self.pooler_activation = nn.Tanh() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + sequence_output = encoder_outputs[0] + print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs + + +class AlbertForMaskedLM(nn.Module): + def __init__(self, config): + super(AlbertForMaskedLM, self).__init__() + + self.config = config + self.bert = AlbertModel(config) + self.layer_norm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.classifier.word_embeddings, + self.transformer.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] + hidden_states = self.dense(hidden_states) + hidden_states = gelu_new(hidden_states) + hidden_states = self.layer_norm(hidden_states) + + logits = self.word_embeddings(hidden_states) + + return logits + + +class AlbertAttention(nn.Module): + def __init__(self, config): + super(AlbertAttention, self).__init__() + + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + w = self.dense.weight.T.view(16, 64, 1024) + b = self.dense.bias + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) + return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config =config + self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for i in range(self.config.num_hidden_layers): + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.layer_norm(ffn_output + attention_output) + + return hidden_states + + +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +class AlbertEncoder(nn.Module): + def __init__(self, config): + super(AlbertEncoder, self).__init__() + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.transformer = AlbertTransformer(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + hidden_states = self.transformer(hidden_states, attention_mask, head_mask) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + +# config = AlbertConfig.from_json_file("config.json") +# # model = AlbertForMaskedLM(config) +# model = AlbertModel(config) + +# model = load_tf_weights_in_albert(model, config, "albert/albert") + +# print(model) + +# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]]) +# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]]) +# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]]) + +# # sequence_output, pooled_outputs = model() + +# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1] + + +# embeddings_output = +# print("pooled output", logits) +# # print("Pooled output", pooled_outputs) + +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json") +model = AlbertModel(config) +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert") \ No newline at end of file From 91ccbae788de11f0f5ffb862421e345b27a20a76 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 21:21:57 +0000 Subject: [PATCH 077/143] Accepts multiple sizes --- transformers/modeling_albert.py | 143 ++++++++++++++------------------ 1 file changed, 60 insertions(+), 83 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b006cbe8fd..f3cebdc3d9 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -5,6 +5,7 @@ import logging import torch import torch.nn as nn from transformers.configuration_albert import AlbertConfig +from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new logger = logging.getLogger(__name__) def load_tf_weights_in_albert(model, config, tf_checkpoint_path): @@ -32,14 +33,14 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): print(model) for name, array in zip(names, arrays): + print(name) og = name name = name.replace("transformer/group_0/inner_group_0", "transformer") - name = name.replace("LayerNorm", "layer_norm") name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm") + name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm") name = name.split('/') print(name) @@ -84,44 +85,22 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): return model -class AlbertEmbeddings(nn.Module): +class AlbertEmbeddings(BertEmbeddings): """ Construct the embeddings from word, position and token_type embeddings. """ def __init__(self, config): - super(AlbertEmbeddings, self).__init__() + super(AlbertEmbeddings, self).__init__(config) self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) - self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - def forward(self, input_ids, token_type_ids=None, position_ids=None): - seq_length = input_ids.size(1) - if position_ids is None: - position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) - if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) - - word_embeddings = self.word_embeddings(input_ids) - position_embeddings = self.position_embeddings(position_ids) - token_type_embeddings = self.token_type_embeddings(token_type_ids) - - embeddings = word_embeddings + position_embeddings + token_type_embeddings - embeddings = self.layer_norm(embeddings) - embeddings = self.dropout(embeddings) - return embeddings + self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) - def get_word_embeddings_table(self): - return self.word_embeddings - - -class AlbertModel(nn.Module): +class AlbertModel(BertModel): def __init__(self, config): - super(AlbertModel, self).__init__() + super(AlbertModel, self).__init__(config) self.config = config self.embeddings = AlbertEmbeddings(config) @@ -129,6 +108,7 @@ class AlbertModel(nn.Module): self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) @@ -166,7 +146,7 @@ class AlbertForMaskedLM(nn.Module): self.config = config self.bert = AlbertModel(config) - self.layer_norm = nn.LayerNorm(config.embedding_size) + self.LayerNorm = nn.LayerNorm(config.embedding_size) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) @@ -182,39 +162,47 @@ class AlbertForMaskedLM(nn.Module): hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] hidden_states = self.dense(hidden_states) hidden_states = gelu_new(hidden_states) - hidden_states = self.layer_norm(hidden_states) + hidden_states = self.LayerNorm(hidden_states) logits = self.word_embeddings(hidden_states) return logits -class AlbertAttention(nn.Module): +class AlbertAttention(BertSelfAttention): def __init__(self, config): - super(AlbertAttention, self).__init__() - - if config.hidden_size % config.num_attention_heads != 0: - raise ValueError( - "The hidden size (%d) is not a multiple of the number of attention " - "heads (%d)" % (config.hidden_size, config.num_attention_heads)) - self.output_attentions = config.output_attentions + super(AlbertAttention, self).__init__(config) self.num_attention_heads = config.num_attention_heads - self.attention_head_size = int(config.hidden_size / config.num_attention_heads) - self.all_head_size = self.num_attention_heads * self.attention_head_size - - self.query = nn.Linear(config.hidden_size, self.all_head_size) - self.key = nn.Linear(config.hidden_size, self.all_head_size) - self.value = nn.Linear(config.hidden_size, self.all_head_size) - + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads self.dropout = nn.Dropout(config.attention_probs_dropout_prob) self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() - def transpose_for_scores(self, x): - new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) - x = x.view(*new_x_shape) - return x.permute(0, 2, 1, 3) + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) @@ -248,7 +236,8 @@ class AlbertAttention(nn.Module): context_layer = context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) reshaped_context_layer = context_layer.view(*new_context_layer_shape) - w = self.dense.weight.T.view(16, 64, 1024) + print(self.dense.weight.T.shape) + w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) b = self.dense.bias projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b @@ -262,7 +251,7 @@ class AlbertTransformer(nn.Module): super(AlbertTransformer, self).__init__() self.config =config - self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) @@ -273,18 +262,11 @@ class AlbertTransformer(nn.Module): ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.layer_norm(ffn_output + attention_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) return hidden_states -def gelu_new(x): - """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). - Also see https://arxiv.org/abs/1606.08415 - """ - return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) - - class AlbertEncoder(nn.Module): def __init__(self, config): super(AlbertEncoder, self).__init__() @@ -305,27 +287,22 @@ class AlbertEncoder(nn.Module): outputs = outputs + (all_attentions,) return outputs # last-layer hidden state, (all hidden states), (all attentions) -# config = AlbertConfig.from_json_file("config.json") -# # model = AlbertForMaskedLM(config) -# model = AlbertModel(config) - -# model = load_tf_weights_in_albert(model, config, "albert/albert") - -# print(model) - -# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]]) -# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]]) -# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]]) - -# # sequence_output, pooled_outputs = model() - -# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1] - - -# embeddings_output = -# print("pooled output", logits) -# # print("Pooled output", pooled_outputs) - -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json") +model_size = "base" +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size)) model = AlbertModel(config) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert") \ No newline at end of file +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size)) +model.eval() +print(sum(p.numel() for p in model.parameters() if p.requires_grad)) + + +input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] +input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] +segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] + +pt_input_ids = torch.tensor(input_ids) +pt_input_mask = torch.tensor(input_mask) +pt_segment_ids = torch.tensor(segment_ids) + +pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} +pt_output = model(**pt_dict) +print(pt_output) \ No newline at end of file From 139affaa8de89c6ef16a4712d24a22a8fea91eda Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 22:45:24 +0000 Subject: [PATCH 078/143] Albert layer/layer groups --- transformers/modeling_albert.py | 80 ++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index f3cebdc3d9..440ccf2bce 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -30,17 +30,19 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): names.append(name) arrays.append(array) - print(model) + for name, array in zip(names, arrays): + print(name) for name, array in zip(names, arrays): print(name) og = name - name = name.replace("transformer/group_0/inner_group_0", "transformer") name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm") + name = name.replace("LayerNorm_1", "attention/LayerNorm") + name = name.replace("inner_group_", "albert_layers/") + name = name.replace("group_", "albert_layer_groups/") name = name.split('/') print(name) @@ -104,7 +106,7 @@ class AlbertModel(BertModel): self.config = config self.embeddings = AlbertEmbeddings(config) - self.encoder = AlbertEncoder(config) + self.encoder = AlbertTransformer(config) self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() @@ -133,6 +135,7 @@ class AlbertModel(BertModel): extended_attention_mask, head_mask=head_mask) sequence_output = encoder_outputs[0] + print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) @@ -246,18 +249,18 @@ class AlbertAttention(BertSelfAttention): return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask -class AlbertTransformer(nn.Module): +class AlbertLayer(nn.Module): def __init__(self, config): - super(AlbertTransformer, self).__init__() + super(AlbertLayer, self).__init__() - self.config =config + self.config = config self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for i in range(self.config.num_hidden_layers): + for _ in range(self.config.inner_group_num): attention_output = self.attention(hidden_states, attention_mask)[0] ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) @@ -267,42 +270,59 @@ class AlbertTransformer(nn.Module): return hidden_states -class AlbertEncoder(nn.Module): +class AlbertLayerGroup(nn.Module): def __init__(self, config): - super(AlbertEncoder, self).__init__() + super(AlbertLayerGroup, self).__init__() + + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for albert_layer in self.albert_layers: + hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + + return hidden_states + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) - self.transformer = AlbertTransformer(config) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) def forward(self, hidden_states, attention_mask=None, head_mask=None): hidden_states = self.embedding_hidden_mapping_in(hidden_states) - hidden_states = self.transformer(hidden_states, attention_mask, head_mask) - outputs = (hidden_states,) - if self.output_hidden_states: - outputs = outputs + (all_hidden_states,) - if self.output_attentions: - outputs = outputs + (all_attentions,) - return outputs # last-layer hidden state, (all hidden states), (all attentions) + for layer_idx in range(self.config.num_hidden_layers): + group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + + return (hidden_states,) -model_size = "base" -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size)) + +model_size = 'base' +hidden_groups = 1 +inner_groups = 1 +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) model = AlbertModel(config) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size)) + +print(model) +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) model.eval() print(sum(p.numel() for p in model.parameters() if p.requires_grad)) -input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] -input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] -segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] +# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] +# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] +# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] -pt_input_ids = torch.tensor(input_ids) -pt_input_mask = torch.tensor(input_mask) -pt_segment_ids = torch.tensor(segment_ids) +# pt_input_ids = torch.tensor(input_ids) +# pt_input_mask = torch.tensor(input_mask) +# pt_segment_ids = torch.tensor(segment_ids) -pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} -pt_output = model(**pt_dict) -print(pt_output) \ No newline at end of file +# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} +# pt_output = model(**pt_dict) +# print(pt_output) \ No newline at end of file From 12290c0d5ce8475e884190b6ba480a8b3e671b3e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:19:02 +0000 Subject: [PATCH 079/143] Handles multi layer and multi groups --- transformers/modeling_albert.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 440ccf2bce..90e9b162e6 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -136,7 +136,6 @@ class AlbertModel(BertModel): head_mask=head_mask) sequence_output = encoder_outputs[0] - print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here @@ -260,12 +259,11 @@ class AlbertLayer(nn.Module): self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for _ in range(self.config.inner_group_num): - attention_output = self.attention(hidden_states, attention_mask)[0] - ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) - ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) return hidden_states @@ -303,16 +301,16 @@ class AlbertTransformer(nn.Module): return (hidden_states,) -model_size = 'base' -hidden_groups = 1 -inner_groups = 1 -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) -model = AlbertModel(config) +# model_size = 'base' +# hidden_groups = 1 +# inner_groups = 2 +# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) +# model = AlbertModel(config) -print(model) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) -model.eval() -print(sum(p.numel() for p in model.parameters() if p.requires_grad)) +# # print(model) +# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) +# # model.eval() +# # print(sum(p.numel() for p in model.parameters() if p.requires_grad)) # input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] From 1b92564330aa2f40b065a0b9a2a94a28a595bbc6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:23:18 +0000 Subject: [PATCH 080/143] Reorganize and cleanup --- transformers/modeling_albert.py | 287 +++++++++++++++----------------- 1 file changed, 132 insertions(+), 155 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 90e9b162e6..c6662cb6d3 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -100,6 +100,138 @@ class AlbertEmbeddings(BertEmbeddings): self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) +class AlbertAttention(BertSelfAttention): + def __init__(self, config): + super(AlbertAttention, self).__init__(config) + + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + + + # Should find a better way to do this + w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) + b = self.dense.bias + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) + return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + + +class AlbertLayer(nn.Module): + def __init__(self, config): + super(AlbertLayer, self).__init__() + + self.config = config + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) + + return hidden_states + + +class AlbertLayerGroup(nn.Module): + def __init__(self, config): + super(AlbertLayerGroup, self).__init__() + + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for albert_layer in self.albert_layers: + hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + + return hidden_states + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + + for layer_idx in range(self.config.num_hidden_layers): + group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + + return (hidden_states,) + + class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -169,158 +301,3 @@ class AlbertForMaskedLM(nn.Module): logits = self.word_embeddings(hidden_states) return logits - - -class AlbertAttention(BertSelfAttention): - def __init__(self, config): - super(AlbertAttention, self).__init__(config) - - self.num_attention_heads = config.num_attention_heads - self.hidden_size = config.hidden_size - self.attention_head_size = config.hidden_size // config.num_attention_heads - self.dropout = nn.Dropout(config.attention_probs_dropout_prob) - self.dense = nn.Linear(config.hidden_size, config.hidden_size) - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.pruned_heads = set() - - def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.num_attention_heads, self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - - # Prune linear layers - self.query = prune_linear_layer(self.query, index) - self.key = prune_linear_layer(self.key, index) - self.value = prune_linear_layer(self.value, index) - self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) - - # Update hyper params and store pruned heads - self.num_attention_heads = self.num_attention_heads - len(heads) - self.all_head_size = self.attention_head_size * self.num_attention_heads - self.pruned_heads = self.pruned_heads.union(heads) - - def forward(self, input_ids, attention_mask=None, head_mask=None): - mixed_query_layer = self.query(input_ids) - mixed_key_layer = self.key(input_ids) - mixed_value_layer = self.value(input_ids) - - query_layer = self.transpose_for_scores(mixed_query_layer) - key_layer = self.transpose_for_scores(mixed_key_layer) - value_layer = self.transpose_for_scores(mixed_value_layer) - - # Take the dot product between "query" and "key" to get the raw attention scores. - attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) - attention_scores = attention_scores / math.sqrt(self.attention_head_size) - if attention_mask is not None: - # Apply the attention mask is (precomputed for all layers in BertModel forward() function) - attention_scores = attention_scores + attention_mask - - # Normalize the attention scores to probabilities. - attention_probs = nn.Softmax(dim=-1)(attention_scores) - - # This is actually dropping out entire tokens to attend to, which might - # seem a bit unusual, but is taken from the original Transformer paper. - attention_probs = self.dropout(attention_probs) - - # Mask heads if we want to - if head_mask is not None: - attention_probs = attention_probs * head_mask - - context_layer = torch.matmul(attention_probs, value_layer) - - context_layer = context_layer.permute(0, 2, 1, 3).contiguous() - new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) - reshaped_context_layer = context_layer.view(*new_context_layer_shape) - print(self.dense.weight.T.shape) - w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) - b = self.dense.bias - - projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b - projected_context_layer = self.dropout(projected_context_layer) - layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) - return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask - - -class AlbertLayer(nn.Module): - def __init__(self, config): - super(AlbertLayer, self).__init__() - - self.config = config - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.attention = AlbertAttention(config) - self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) - self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask)[0] - ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) - ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) - - return hidden_states - - -class AlbertLayerGroup(nn.Module): - def __init__(self, config): - super(AlbertLayerGroup, self).__init__() - - self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - for albert_layer in self.albert_layers: - hidden_states = albert_layer(hidden_states, attention_mask, head_mask) - - return hidden_states - - -class AlbertTransformer(nn.Module): - def __init__(self, config): - super(AlbertTransformer, self).__init__() - - self.config = config - self.output_attentions = config.output_attentions - self.output_hidden_states = config.output_hidden_states - self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) - self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - hidden_states = self.embedding_hidden_mapping_in(hidden_states) - - for layer_idx in range(self.config.num_hidden_layers): - group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) - - return (hidden_states,) - - -# model_size = 'base' -# hidden_groups = 1 -# inner_groups = 2 -# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) -# model = AlbertModel(config) - -# # print(model) -# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) -# # model.eval() -# # print(sum(p.numel() for p in model.parameters() if p.requires_grad)) - - -# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] -# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] -# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] - -# pt_input_ids = torch.tensor(input_ids) -# pt_input_mask = torch.tensor(input_mask) -# pt_segment_ids = torch.tensor(segment_ids) - -# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} -# pt_output = model(**pt_dict) -# print(pt_output) \ No newline at end of file From 67b422662c7002319e54679a73d654ba313702ef Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:33:53 +0000 Subject: [PATCH 081/143] Documentation + improved AlbertForMaskedLM --- transformers/modeling_albert.py | 85 +++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index c6662cb6d3..9b3c51fe25 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -4,8 +4,11 @@ import math import logging import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from .file_utils import add_start_docstrings + logger = logging.getLogger(__name__) def load_tf_weights_in_albert(model, config, tf_checkpoint_path): @@ -232,6 +235,70 @@ class AlbertTransformer(nn.Module): return (hidden_states,) +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -274,6 +341,7 @@ class AlbertModel(BertModel): return outputs +@add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(nn.Module): def __init__(self, config): super(AlbertForMaskedLM, self).__init__() @@ -292,12 +360,19 @@ class AlbertForMaskedLM(nn.Module): self._tie_or_clone_weights(self.classifier.word_embeddings, self.transformer.embeddings.word_embeddings) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): - hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] - hidden_states = self.dense(hidden_states) + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None): + outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + sequence_outputs = outputs[0] + hidden_states = self.dense(sequence_outputs) hidden_states = gelu_new(hidden_states) hidden_states = self.LayerNorm(hidden_states) + prediction_scores = self.word_embeddings(hidden_states) - logits = self.word_embeddings(hidden_states) + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs - return logits + return outputs From fedac786d401a9da31847b45df842bbee185afe6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 14:39:16 +0000 Subject: [PATCH 082/143] Tokenization + small fixes --- transformers/modeling_albert.py | 2 +- transformers/tokenization_albert.py | 210 ++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 transformers/tokenization_albert.py diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 9b3c51fe25..a1c0d5f610 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -298,7 +298,7 @@ ALBERT_INPUTS_DOCSTRING = r""" """ @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", - BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py new file mode 100644 index 0000000000..f2e37222f6 --- /dev/null +++ b/transformers/tokenization_albert.py @@ -0,0 +1,210 @@ + +from .tokenization_utils import PreTrainedTokenizer +import logging +import unicodedata +import six +import os +from shutil import copyfile + +logger = logging.getLogger(__name__) + +SPIECE_UNDERLINE = u'▁' + +class AlbertTokenizer(PreTrainedTokenizer): + """ + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + # vocab_files_names = VOCAB_FILES_NAMES + # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, + do_lower_case=False, remove_space=True, keep_accents=False, + bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", + pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): + super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, + unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + + self.do_lower_case = do_lower_case + self.remove_space = remove_space + self.keep_accents = keep_accents + self.vocab_file = vocab_file + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + + @property + def vocab_size(self): + return len(self.sp_model) + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(self.vocab_file) + + def preprocess_text(self, inputs): + if self.remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + outputs = outputs.replace("``", '"').replace("''", '"') + + if six.PY2 and isinstance(outputs, str): + outputs = outputs.decode('utf-8') + + if not self.keep_accents: + outputs = unicodedata.normalize('NFKD', outputs) + outputs = ''.join([c for c in outputs if not unicodedata.combining(c)]) + if self.do_lower_case: + outputs = outputs.lower() + + return outputs + + def _tokenize(self, text, return_unicode=True, sample=False): + """ Tokenize a string. + return_unicode is used only for py2 + """ + text = self.preprocess_text(text) + # note(zhiliny): in some systems, sentencepiece only accepts str for py2 + if six.PY2 and isinstance(text, unicode): + text = text.encode('utf-8') + + if not sample: + pieces = self.sp_model.EncodeAsPieces(text) + else: + pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + cur_pieces = self.sp_model.EncodeAsPieces( + piece[:-1].replace(SPIECE_UNDERLINE, '')) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + # note(zhiliny): convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in new_pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + new_pieces = ret_pieces + + return new_pieces + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + return self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index, return_unicode=True): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + token = self.sp_model.IdToPiece(index) + if six.PY2 and return_unicode and isinstance(token, str): + token = token.decode('utf-8') + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return token_ids_0 + sep + cls + return token_ids_0 + sep + token_ids_1 + sep + cls + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1] + return ([0] * len(token_ids_0)) + [1, 1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 + | first sequence | second sequence | CLS segment ID + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + cls_segment_id = [2] + + if token_ids_1 is None: + return len(token_ids_0 + sep + cls) * [0] + return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From e3ea5d1d8db36091ab840328cf6691c4f46f2e89 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 15:03:30 +0000 Subject: [PATCH 083/143] Docstrings --- transformers/modeling_albert.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index a1c0d5f610..ad8b979cef 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -300,6 +300,25 @@ ALBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -343,6 +362,27 @@ class AlbertModel(BertModel): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(nn.Module): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ + def __init__(self, config): super(AlbertForMaskedLM, self).__init__() From ee20201d339b62948acd05783efc48c4e2b466bb Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 16:19:49 +0000 Subject: [PATCH 084/143] Tokenization tests + fixes + init --- transformers/__init__.py | 5 ++ transformers/tests/fixtures/30k-clean.model | Bin 0 -> 760289 bytes .../tests/tokenization_albert_test.py | 78 ++++++++++++++++++ transformers/tokenization_albert.py | 30 +++---- transformers/tokenization_xlnet.py | 8 +- 5 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 transformers/tests/fixtures/30k-clean.model create mode 100644 transformers/tests/tokenization_albert_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 5c7b0a6197..152d520e7b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_albert import AlbertTokenizer from .tokenization_camembert import CamembertTokenizer # Configurations @@ -57,6 +58,8 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import AlbertConfig, ALBERT +from .configuration_albert import AlbertConfig from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling @@ -104,6 +107,8 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model + from .modeling_albert import (AlbertModel, AlbertForMaskedLM) + # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/30k-clean.model new file mode 100644 index 0000000000000000000000000000000000000000..c91b8acfa56ccfc80e1cdd854ddcaf9b6c44ab2a GIT binary patch literal 760289 zcmZUcd7RhN_s5k&L$Y)#Q6X7UnrY9TB`SONgwHlJpPA3r=l%K2G+9#FLzbju2}KB5 z$`(rYuO&r<5?PW$2?_Z<&pq$Y9goNF^VdDE&t1+v_uTE=bML!s=d$L9=OXbVWgC9& zwk=b-n5hlvrb-V#_M~AW{`3FvVaK25=~iXi4L^C{u%R9(D{FF~MQ&C`_@MuU_x(?} z`+vfRl)^ieZTs~Vx0R?J%bLFWK#9=5#?3BKWl)pHO8a&@EhuYQwjJR@bwc9fj$2T+ zLs^qmS-d* zyS00z^k;P>E-fc`vLRcP^H)HUNnC#ZY(XhiG$B_vUFh&b<-!54mWr)U4bbd2dY)s4l4WVWUN|(=K-uSsv~s?xpJ@f90w6;dKn@62YqXhE3UJD zj*@F85VA^CC)3hx;rlL0LcZK)iT{I!KBOa1Ko z12jZ^8Ut8O2uXp^&)>m z()ek)qwS}qoTvw_6`ctXcGdcm_#^qH)5&aNz_PiJ9v zY5A8W?qX>CuahCER)h5VnUGh2jJ$lws!}%f33=!iDB6P#{P?zCl_EwXv0B-2zNhQ8 zLq8Y!+C`Qh~xv=F2AnS zg6RmN8wW|RK{~R>7lbYIc(y(x`~3}F(!D&ow_W@7r3mRPbxEBJZKX#Og?jBM*@Zqz zeUffF<5OtU?o75`X4U>w%CeY=qi$R8^#Bz}FdWYH0E12{A$?Z>7+3WPa%{cR&rTRU zrB#}c{S!+T8l?Sazm(>J3fcSMUolBTiN_q$8k)~JeH6ZpN?c2zU-O6^P*)x$sGMkB{WZ0!( zmSvELWvSJOk9;a7puBY3ZPS9X)@3b8Rm>zAJOY(kurH|P9c2iRp;ex)PB4viUJh-7 zL`ecvYPa7?2~?$I+X2uz*alKHjG&379_8w_5Ykk!Dl3<4_j@U$Vy2Kx$xm$nEOu3b zsr(Q~x>bcd+>4O0l&C|g9vSXas`N85c)CxSt*n$??)N0(k^e3B$?LQETG@Mvrz?}G zv@C1&M@dh5%58^3SXqTcBrSWyp{=?BOr~U?w|#_E7FD?POXyM^^YUen&8`D!!8>0_ zxYV~}5yp-)MuUX8B{Q*vG#%FnF#cA{QCImiG)e*8sF(<1QY!}X#-C8eUv-l1z2a|Z zE4~^QEU;CmBPJ_{t}%&T0Vzn{ROqj?#@9LeR3z!(Bn{y9gj`x&}0&_b7|YYGKF?1dXqN zj{f7jknmM2_Z{$8smLG^39=bE;s}tiQL@=u**qB9ipga2NNv&OK0+#4nUIN#8ne(2 z7s$PDfY`#Ymz=%HN6%=pyx8jRQvQL`nSP#5MCiX4PJ*`1h>+32ylkEdWZcJ5?XCAh z8b`TQgogd&X#jcwA{_mi1yP<&iwRmq(mF8A*y8DGomAPcxcjWNRBwSL?pu;QpRYB`OPY%aPFGSjC!BmkrI&12VOVv0OuqcTa0*$KNv|J#4(SL>fa6>ITPieD zOh;j3kbszA-0a)B%G0R`L+Y*LppElbHl5S{G2}`Q1=`YR^@y8%s+w$dMjrkE8bS7w zCzca3o)ZXWq{Ci|+U%h*e$(;e(a5I@Cup1zETS z+9VK7RM8KHZQG=jYd(>pXOl;#GB7H}5_umX2%}YUY6lDTk$hJ|R%j|&C@?RM831C1 z7P48{coZb*85z$yfuQy9-qQU(ke(0(9QuX*Y2kxqZ`8aS=8<2k#AOQ+Jgv`!neT67PF3sBs;nb(2&z$OGv} zp7G|MD1fn9&^BvU1#LVO<-WlG$w(8GifPc+IS>MC$%2e|3<#aoQK0f&!n9XW=By$V z){WtrWbPta}4c5%ZPt1rJwxvc57rHC~}BGxzebiG#Ti5;MghpI%z z^~Mj&e3U?^ym#9{TZae|xkAEq{KxkJgt^|bv3uj+46)i8Rnnq2pv|eQxX(Eh%H$St zbNU`f16boCSQ#=1(rVM73y6_}p~*Zar=3p7G;;}Dijy<1`iIRMKhU_tX)BU#BF za7mtjt!mj=A*;hx?}CSPoybB?4jK<+W!DtZyPaL;uNF=nHxBKG#dh}0UlDY_7? zRwMf#0A^)Ulh?XInW!qMs*JDjDdLHkRpgxBK1D?zIqe9aBASgi$dsd?tx$G+Y?jJT z{7(c{e6ESpU%Cj|W#cSP075OH4h#sNddV! zO;)*lLYE>;a+BZpk4lYRi!+zk&v}WB+$8c z`)#%_DL|o^&wz~B76kqnF}jx#wAeXzGI_aO0V_)_9WB~GYrI~rXQG9Jf5tq#1uxwA221uRWv0+m^t@M&TcTVxJc)2_eHc0{K2k3) zJ>tn6gV#lKAx%^^KL^t76(BY5!R-9S7RK%)&3+^t%DsTutsuYr>=SdiP{<;)Q~w3B z(yJ4xoa`VwloF+F*Y{&PkWd%vBedtw9RNa2hQSNUeF_c?w7L%|Vx6#`yj0sx8Z5p=r9uRAYc(N`T=sfr2 z3@~Gr4H%r!$nIYP zguPrBT$~MI$N2%!n!~p830bxt$ZFqT+HQa_1`4{4t5w=^O}RBeQ32}6>B#C9ElU6C zjy#a(S^|UvX@42o8p?{oM4c(ncmvx2S?(!anr*wMr|C3tIiowYm6qj1BO}M01Yi|r zO&mvL+0#KxA~8%8(;7+L-+z8K|5}+rE)mlWd;5q_P+#33HF=+7CEcIZ;FvleAf(UyncG2yG%`%sBLB|8GIf$`2`R@<4=)jMb9>tW6`a7}J-k?`|`EtVmi{g1bKIv*}sR z%JUwjQIdK|KW3p3Qu!<)Q^zb;GB;#p7J*rDY-%WO?%M#yePt1T4td{4sA53P$Xg#a zMqrgH)ytqQ*HFDsA;rQGfLTrTzV$agS`;HnCV?h@{e#a6TUH?JmA0m55t3rpandG_ zBr@(;VdZZis*HJ=+oG%_$co-_{_apFC)U1cHDWf%+_r$B_UWqd^Sz)O6*R_rf7*UP zR;aTt)rA<^cVRh5m?52Btv1K>29mO!*>lF}(Wz|C zZIb^!1~}9bj1pHI2Wf(@&#KY(`V#=CSf0b|GYA^j9ARhVw_(sGV&-mUBUE^=v%wfq zb@;D6*T+xkdB|fIKwI8v*MrAj1VG+wLPkd{m`F*7%LrP{vDxZ=q3UuFQ_XZzy4(mA zszQ?OYX3==v=EWWVUQE6in~FrUKx0aJ?68h=q2+PSdw%^9$rk)a>MMJk!wHkG#XnW z-zeYE>ey@=g3= zQIip=+vxv{q%Z+8Vl>;a#2iBeHsa2nuGabFg#)0Cmt^3^u{KBiCkFlGos*%%VW^nP zX;)f)IuOiZURZgyPajFSGpno41F`C|*<(oQ838Cw7A#+w`ec`ck7Nf#6Ker%SQ;Qd z6(B;17HOZcS9ltU_LSyRp^a&0eF&sR5$hv~c#&PF_ROcJ0a~7Qa>i_ku;)hdm_OsP z^Bf@KR6A~!9Q2Hj!}(4j&q?9JML@<%&+_vA((4UqE1Ie>XvuAj2!~2)3Tw8MI9B1H zP}Bu!AWJ{>={U*LIdseSAfcR+38tqzegr_O%miIF_}F^D0+4iUmbO2b#G`Y&D!I#WBgx; zy07#--u{Y5WcpzLS6b)RcZNY*p2?)lI@2OzhUqfgN2K*QaUB62j!kODzYwxbSt~}c zSal?wLra3{Bvz*=xV1$gTVSYrQ~^sD!BqP|0@CVbgBig?SDL> z)&n;>KBcJXou1bwIRQIxG>{iOd*bV$ZCs56K{l2&^^M*DOzXN+`#Y5|4sGq&=SH7N zjFq$O{UEfl$e~h_?QerDn(Y(ivkg?M=Ntel37aTtV+*OmGr-1UB*g@U&7kQ^9>PxH z7I*5e1grKyu()Mj-5ZS*L{c8@547jN{ub1qsI$UEaXn zn&m(!1FV0til8s7x19FXf9Udlvhq8hI+xE@grM#!6PRCnHJ`ah zaV0B7-AbN59mvG*c1D5By5C2N=54n$X&$q^p{8JAUwteUJ| zUebuzOt;fA5Mzg>9F?#5#7A+RKPAgP2eF*diyEd(Nw4;iYS_t9){q}SLQy;A0c-gq znAHqDiOW)@5eUnwzc%;`botGQQMM7prj%Tk5rfm|lw|(^v=JhiQDd!*MSZsdhayw2 zwx@U5xkS^>1!>+I+BlA>nGIf6w*@lgs+D3#2rDa@)-{BlXT8-KFdS8LIp!o%{nZ7` z3euIbr1yt5X~o&##-(ddAEB5-dNU985u#aL(cEwtfaMz6Ap>cC9GJCBT8)(j=|0$J zgPJB}-btQjiP2ZSy5PTbANlJ-pF=d_&dr{Vf*5ByM+dBFe3S&%PL%e>N}owpGM|!E z$|s3pXpKv|kv>UKOsiqM%Q%l{qp*CMbt8z0R*kbd9L&4fL+Zz(Ej{3NAnQR@WISv# zBqJAw23hkEVUv9%!(Ib}WL(xh0caA5V>PSF$h0RxO!Cnb*1SnCL8CkL>f>Mbe_=I1 zoQt5XFpf*5&)bkDaXUaI|GPf`+o7yUoou(5kd@@TyD~CQ5u3exl}qk(DASX2S--+Z zuFOZ&bh_^<5HDsn=94x6s7`04&F=(_4K3+PRuM*TVxrP7f_V z>jx0_M?H0ZxIZ*fsgda6giI=&8>-&+KguVj)6rwkRRC6<4^W>IAi_<+0C{96lu0~M z%(G*o_V1qzRvI+eOrhjG&H%DPf{|D`6Efgz5L*B-vQ)BHWdx#|T{YY%R+l{O;*Ujv zLcue>;=@tn0oGvbHd?2AB36Uy-69`kfWm6o)@D@RN3sPop3C(>#=Ubk(HWukN-(Qm zJTE&>fHIXuJ7f7|6mgGH&=jRwj=vScB+GgzC%aFAHdTnSgYI|-WXMOpLHo@6_ke`G z0OJMwS!(ddOu(>R0T}Nxpz9w6wVLZnIWE6E?sH?=m|<8S{FINvYKx^ppQi!na11W8 z;Ta1jgR~UpHYR2B&VY5y^B~4~E>cWM@0Xxa!fYTbUh#ji9WKby1<+RGynL>NwQs-- zuYF^;_koO2hQ>rzcKe_)1J%Eul~x5$OS8rPuPQZ$y|conWvXB$-fbm-Nw5KP<%*wt z{CZsuRR7}t<_t-?Y=X4*>e)vIZ-Fv#r<3tG3-8E(kSyILBe|`OaX8?oS3S~Xmy#X? z+J7|GVcU6(HI*%hy6ymG5_3D9^ILeDMJ(>^EupC;UgxWK@qg)Qirlfgr*&g3|0->j z}r;t@|r71@MgdJTCKrN1f zHi6_5x-pVDM}t^VY+)FeDe9ELU@(q>;@Oi4hP{_l0M*!I&Id9^P*;|x_f=YS`q}XG zfBw_G>@^vHus>+mAD@Lb&2|TkwjLgr2enqiRs6{!lnH}ld>LH_Y2~R{@jk5wF!eh^ zKE4#f__uycp>K|}2zVq^$%yNG@=7MEyqr7{fG(ky&LwvfHujt}mfd6jB^kJWyw4|u zRd^pg3&8lQC@+`Uk3pJfSm-&giR|M(iMkZg<(>ebEOq10&mnAOv5)K{-JgZF?3le| z%e(*in^VT}4=t%S$aS9(wo0;`tt#rNRri(lKl^wYuHjJ6F%VP-a@$uPaTfMKZu%O; zWSq{+h;N~2V*IP8tnpFv>YPxJSJ#4A%@U*7j?!=X`~>8gFP9HCc$%r7iG8C_l8+?S z#v^}h0y3szYG2}f@-Lr=gVDHbY_V%82IESR1Dua{1+kJt&mnoLjgQ2sQyQE2L+yQ} zG`ABXtbuRe6Uf+O(G!=)IzUrBjKGho-K7QvVLd8C%@G?hL6NS_5Ww{2)2RLlx?4hM00=s1wD zf4Q^yyc{qH$n-t#{O7hC0$|J))N614P-s(}fF;7{Qz4DlNIY(Po3hiu&^w(l=MN)n z`MJYZIrBmQ<06wt#NjoW1qnyZ3i&`O`=5D+vp)GW0HMb@KkU=8ZGVK6*z3W$k_`5>>l z|M^rPnu1$2>FGXx-n>;_QN+ZP!dqnRgFbnzn0Kd<`#%a)8c}q;dxZj41ilMQS#i1P z>Hk1MPS@fao&};*_9&k`Pta!VFjE$0kG%k>fvA8KD>sZ@d z{|1bmz(7Yezxf0p)apb6*Rs!{{ou*@rZd_1Tn*}#D#oNgYMn>0GOO<1 zwB92)fT((Q{RzbKroIt5=nv?S;Vg^HKmFfnUBylL3%a!OaW;1HSg4@YfDsTcv*cC) z^rWB{I?mf+w~|oZlu3e(3L(KvQV!aQpmB|LTKyS@w*=9|dYfghwh&g;@O}ddB(ErH z1;!IPrnYYfV6|XO#T&3awAF%jq})1HeN8Omh%4Iv&V($N4z)w?w7-6kUHzeO^H#v3GorZ-!$ld3 zjzN5R-bc5i)CF}Oz&1vAKj~77T$`1+1}!TF^}McLs}w>d2nW*h3ZIrG6Z1gxu>fHw zBmv_#Q|0vWfF{OxQm(!l%ETC8XPa;hq$)sChFt3tM>sv;5E`MLub`kO>tRd|SO|!ypQ4xn?0KDUa}Q2J3C{M?n?e`1|0IFn>B`0Rv3urZ6M49n94 zo>_A|jT2cQdp-wkT;ocgw0`#SB0%Gs2?dARKVJ8#@F6J3ON%|N-bymA&p}MaYPZ)NZvG1(xUAv0R)hAhnrn3dv3fO-3C8SlNHh73k7ft?fy`S2 zWc6dG##QAF(AIFY{-J+ESw*-y%W2Br3RpopOsFf#7h8eCf;aIsStW<>3S=ef-rSjCr7f8C zmpTT++mu8$47mI6LD0%Y)vz+IZx3L?tt@J9mGe7*7$=b!*EpFs&*=mdj_W;T!@f`^ zrCJ=>W#E3$VO@%7tDJw3k3hGH%Yq)zp~{)<;kF8&q$a|37Wu0efN73e%FO4ibzeZn zF8t{wuHmpln2C+OaG;>gUpPMK*_1krZ5jYtl5GPG_w}IwR(@b6f`<!uPSZYMMFJZ zF`$3BbUoG6%w2e_D{btlS3i{u$2YN5lt+}++l#-@Nws(Se# zft|z0iA8KI99i$7N_J-I-Lhm9P$+vnp$O#Au^<$ii^%a;5wtwh%<3umaH7wEJPYb% z`sY+2?&D?(6~=L0%8Kfh%DnChcrE5%46@ZJ@9}zJ{49s=X^RB#3Yl_ z9e4e6&{kn{+2X3aLe}b)46byIxcqOvPZrgM!oUTd4w9oeym|}TDx}T;)w1oo9;kHF z+i4#HSV7MFn%QvJCqQ;Mq%5QS?tpLPx4>b=*eiUeq}7~_rTW+0yB>t@p)We@vXQXW zr&8^TZI!lSumDDMBieP7kHRrgQqKL|)2U=Gkad4U(=P0TcWox6ex+M>b=XG1&T0;7 zWfZEJQ?K3uTJ@*DyuBkqD~`R8JMucNrH{=dT_Jz&0&V<9^(?DecG)9L-&a<*_mS~g zWQTl!r%BvXCLHK#Tsl+IsT|tc#*H6(zxRRuU{-*;)5)2}VIVd<;Ag>>)Kz2Z;h@GV z#&zee^Z1cIVj&T$&SX>Bss_1ppwE!gU}pgu>Jd%?l8iv#pXM{uGo(1HtPjrsGTuT5 zw?M|94Q6`5T7xzpIvfDmvAFqFgES*6J!yLZ;ZQp;AM5x&QlYSqu$5&FxDbF2QR&=U zURBB(g%h^^GCb;|?a_3L9Re^Dbiz4Rx}ANGI#6THttu|6hqeZC(`8oPyu|0l_Q%%I zZN~UmdXb8DxJHpsx0SzkW1&ljsrZCochQPII1U)Os9$_Sve!eobuZ67OduR;MkJ{X zdG}2osnhEU_%~1T2u3>H%#5B4!Xn5i&8t%&Ol0b%diZomYm{8I4u)qaU_4d~=vOYs z-3N^fv(@s}41!j=ZdEV|-#6R-tcpm(;{)A`X&4IFPF`;ln{O%_K!m6v@w|^Si z^3ofybfQi3L40%A1a|rYFk?2#p~9#|kh%?3;eGWEgtaKnG{`NGpB1u9dAdVA)sydo zgo9{xo+(oe%Kbh7w32Z8h~;@uMD|+jlVXQZbv^wf5ah094gJKY;BsItqUWJaz67%x zVU|cp#VVfzRwZ=_8Tvg4q8lBNBi0ZMyK0dgBwVyu2V`7u?VHsd*a7Q(9=hP8?G|kS zfeQ_vw#lc!=uzi9Y6kxXW*Ji`Yw5V``@7GCYrc7oM5q7Z5snU3CI0vmBy4tkb66lM zVBF{y+#N4%_qUJ4+a)OG@+NIc))<@@BAMi9voBt-EqH0FQtxM7K(19VQ~~zG)l$Aa zfVECMp&lZ#Z<$A!V{|`t@{S&H*RccHucgn1mw=GbTMAe<(sfQSTOQ(2rVsNRI?_#JaVehn%Q@Htq#v&8h}usow|Msxh*tUM>Ch^$}3*26lOYZ_xDi9kklXGv(X#gTtHL6%E9c1yKy&YtCl`9K($`!4x&v@Ax|Gd*b33T zn(E3!y@0I4p}%GQpe(z9@mn4{9NMy@uTp<$IS|C8RE?#@Iq~*V%u;c;F)rT^f<|Mb zS&_jO<<^A&RyP&_m_agf z%SS+D6GUYBGJ>TsCaz|>23hFyM1=oVT%P&~%*s&bc=_>XpAVKob`xL!;Q`#V7=|AC3m_b0^fimP zJfn!^!P8=l0)In?TH!9o2GX`k+Y-b<6T6>4_HPPQ5-C1{+d)zp9V4F6u>DobT#?%e z((=cmg$GAVXv;s9jOKNBRoBWxmH6XoZ3naivf8@0LUi1HdQUKXNJdKiUW6zvg{Tr8 zd}3C-?ym6t9ev_FHV)_K@lY2qV?!k>OZWH5Ftxf(Ui=^*g^4gDYm~NDQ75KAUhd`r z)|Lzf($8nYDwPZ4Ol1Qa<7hKUkHZ1N)<2DyFXlGtJ@fr6Aub zZS0|4MLq7l{Td*OR8Qb#w-Po1Gg7Fr;-}j{;4Ph%4!2uGJs&&g4#JkNt3qUoPl!gU zP4(C5AXXA%o%6hU?s1^7A^ORKbD*?b)}g(gB5VTZp(vFamwn{qxu8~GCV5tg44BIm zvw|6v;}Uz;a?u09zn=HGIF}Dw20izJ&xMJtpa&sx>MLN>ihCZj->_h$T5eiG&;+hy zHg!q@j5+<~t5wj*NnLDr{D!azHN|PY4uPE;AvNq}nCxCyODtm&ewetJ zo7O1iTDc(2))Nky;4q^g{eSk^vb%P2?)nQ565{^msT#CqOUGo}4N%4&9_$4ft~9f< z+mfBR5yFN~%$4%mCLgzAK;Lqi`vnU7MwdhPcRCK6y^6`1g#+YJs11;ILzmuu^Afr9SdS|=CZTIj`t~W=;k2k_Y(n(lakW| zIxyuFP%BqoLgVZ^CMORAGkG&7T&q-xGgW3(ey5AUoDc(<9U*MxCK?hlvI^3kGhywT z&yHd)!l}MNwy*Z_QtT-1P}11Z(^$OT*o(`O6sWZboBvn@&ynU?AlrJ=QB4(z`EU+6 zlt>Pr3|Xc?m{gZIKu?bHar67XTMGV)PzXBP>?pZxPRwJTXcZyaQ#N0%f&~yE7M^hw7IC8u#iG?gNEm~4BO%MVTWC}qzCT5mL|DtgP@ZTYJtF|~NpSnoIU7{dowX=shFntP1irkms{ zY25Ma=&(UY|m{;a1WV*}g z^FJ5(2x{w5QJoqGGQHygIJr?t6Ej{OfefpJwnEgLg(h5^Y($vq7y>t>d=@&x?UwTq zIIz(kd0!sHGS$N)c{_kMIpH)0<3ASwObygRge#A>&!bm>ny|tb!Dxx{u|6##MGUEN z0AZ!w5d}AV8X=zIbvIs~zLHqzFN59h8Z=dK??CBT*Wp@^={ZGOU8jyw%&MMZsA9Nz zRS~O)9%snW*ZCN{Ql-nWt&Io^Rb2wCn?M|^tFv#)0XG6zXVfe6v1AUj8jFywH-TGi z$s3!b&S(QC0$PcTWj%3S_PKGyOZIo(^4|!3WWnvw#0`L_+d!VV7bF}Eih1|U-SzhYSv~YA)j;l4z{IFaqP%)aG`-(PitF=Na_>w4 zV<(=)fR~V?AM{9|PB^>-qlh(rViaSIT_N~*mXGHiH;v0?Mf^C^# zA!Weh046dXA}dJE6Q0&Biu8UG+UjOo+EFq=A(Vq&UH1$@V_5eZXyKo8L2Tus*O8LE zMxiZp+_S)8zoB>TKOfTAVAsPrX4jWJz~yxM&6ouMxG%e8s7676)&HZn09K=U27|GX85YROhPY{uUr6 z1AU4|UMTa4>{ud@UONNnyxU8TXho0~*Ow4S?dD^$UUN5GH@60|TJg*cqgAw>2h>s? zNL>d2s}+x?Ch*7(GQWNU85f-5t-{_xCxe#VU{`3Rs`8MI5~r5nX>U zu*pU5s%6|{e)b_gKX<-B(>|_H*pg@{!rQe6h%v9%>uRN-fN`OZr|Z&qK)Hw1w@lqb z->mQ%v53>zX{92Tksj6sQta!aV5!t9PdEm|8VU!B>yGvRGlZ*Cz&|H?B$IWXxfh=X z!eWcXNFc4wfS?)Fd^29dRvL+D^SZ*m26VPieQ|QYMG(ua%A@ah-IlZE(Qd) zwTf9L0lS_~Gkl^13sa0>O=g0ap!D>L>rhVulxmA*zvolXVSAyjddBk6LLl0g4ZFl& zwXoV)hb#)iC{^b-eKNP8rkM|X3n*N|gl{E)PEpjjR_9>nzcT9`FsqwgNtQYP^RZG9 zuEFY!?Pc$S8E5P|-PZBK4}jX0?TSxfByM-L%oHaJ^q+77=qA&b`s{)t3ap<)nDB+i z^f_4fxWsFUSt`!mZdeJ$NK=nf(JF$zHF`-zX%nOFR|4txl?S+s#13ST0+v;(CXLc3eVoj_@C_J(*6q%<-xH?rnxq`LmY@~Jtq8`kL5+a3F3FGU ziDB&FXb{K^KSNvL>9pMSr%z9BL33Je0kFXe3mgZ`YOIaR9)ANHllX+FFX&$X_>`)0 zfsEVgfv8(+22J-Yl^o47alE9oRf~MN^{wZR&B2U+#)O{Tq{&VI)RV=jEZK#y70M&H zY8}0ESCCRa<@GOjxc;xU@AF{Wi0g=Zo9~PPptc6r_fTvpNiR9( z2=G$dl@H(+MkC<79AvQqei))Q=-;zPf|(R~giu{V4>=0NG?F`!6$AbMTtLfF&R55R z&`R8OEI*N8IEK}7GCJ7Db@YC zguK-=`L{|7s}JVFBw?!>&(6wGHISB*yVqY(+Hz6@ryjIUX@u;GL?E|hiDlKdj~q13 z`D8WDvo0ljDP#?xwj7F{SOlQ~V3{ttR;f^BSO9cufHX!JKY3gC68|?gd)e+%NXsp% z8_{aHqY;Vg1I~FFIfgi;j$g$n!^CmI6=23GgENtS8S68uw0HgPALpY4xYhGA>6HM+ zSryj#TdsjLk?8KD3Uwbe0SMu$W5(qgE~#fl?`|^uMgXfxPH(m*Zt^sJuU_`M724Qz zFQ)1~v*m4IRz0(5O2zFUmaRHyG{`gsLJ1@|F66b^$sVcX0uz1u`Kch5wV4c2>ocZ# z48yazZeOUFv6!vJ-mq>aG^)>hy<(OHE0gljV+4(bRJO<}>vIK&hPVG+kNf|ax)8(B zPXbt$Y-rf6D(wbXzQXg2|1YiI3i@)c{ikEoKN_}l6$9`)_-P)rRUtI$1ai%@VAcZe zu>+FLeGbTY(-DSE?Q1?C7DRlA=?!QTmwulJLG1NrV-l9@897CPQbVvX)U&*Ki+vmCu981~gRqe-nv@g&fU;_^`!-j|BmV@W zrP%ZSqCpd$8mrSx{FiU>5Mw220=Z5hYX<1-)DQp!3ZR~zIb zC9M{9xTxuqMVGhsIdIUX`;6Nau}W~A2`1b6L|iiG8d5~gYUiUQIS!EX_VhGP>;vR4 zrNgFSDb8MRF955kdhoCq|Fkm*vaL(XYy0`sjB!uJ z)*m|D`w=+G0-l#X!&Ti-D}snW(mpL0Q3-*%-6JU2%ZY=+M8v zJXiy5b>Rj_7H>_~335U$sEJ)~#o-!;J8=puRC{N=WEfE>EGsE9^H5d=*3ShBTcwC) zVW zJpfksnruQQC~0lQ$ft`kHR^t6>xf7Rd?*|GOfr(5eXh?C>hkeNj zGr;Ksd=_x&Lj+AAyphK%tB*okv3liZ;cQ52MZ4#tt@5O$=JAm?G-zBivi9vJmpu(& zOz457cH_Cv`ZzWCy2*oO z#4tHjau6rG?FelmRo^MMIoWP!u+olD?+L4(T;9q@4BT3hb-~`df`ubgk|r`Qfm0Q= zEZuz>O?`7)FqqUAh)!!y7(L)L>h8S=8aGw?tYA$CXsaA9`Bd(dPCf$eBYYHQA0Hu? zWD1aOoqdFW$E0P9(%Oz}CEIl&94Z1gpv_fjxq_h%X+PEDp}GUWLX}Tc$+-tYTHE26 zmSyB#)QIrq0o@u->rNcXVW5iB=!pCuwhRmq+KuuAP&bwE6 zK%Zm_WO+3Jv=KSB$vc1C_~2mM6Iz0Zh$U)%BHICqf%r9Aj|dLLxVVSds2# zj}FE^C>D+btli|Q$7|Q6G5Z0S{f01|i^lWU| zgMh|1t9?!s@;vGXD$X5A!vN?DNi-CJ5?eQ{zwFg@#bp{CJ z^jCZ&PL5z}_yPdyusD$4w+PBts<$+G4ceGyp{^&`i{J7HcD}nZt3?~210&%-y!`eVJlnD>*TgZI$mUE zOE34SosA`s!e=1XDlxtBB<(&2uo~4Saadp&obm;b@quC3SvcgU6+Rt`Nf*0$C4eb$ zJX@5PGpaxf-3Z3-+E9#M50!{f*kk*pfxNEUGpQ9 z?sfG7@^9;Wd~B}lj@4W;`6ocD7QGigllOk{0KR}IbKi{smLqr7vPvO!-d(<0gP|h3 z1{?G{K&Z%i_Hsuf&2dm56`T1#>pVATE2w3wPwBAisA<-*)HLc{q^FagHTMvvH`WroIkN+hrNb`9T}e5j z%tzv|-FatE+8L-MA6@ed6c4cKWzDlo8vv6Jmfi_%{olnr*8$|*y#RdQ>LcwtK~qI8 z>`vL2pj9V)2v+YO%~Z6cM7RvAOWeZF9-~j|gT;JT{17l>zlt*oT6;}z5YrNUY{6XP zcI^XbG;2TnX*5y`{r3J|Zq&7Bjte5P0oKNel-AvE-o#@lDL#vl|LwpqXushq@4-{fJzzij;hY~hf>LDtt z^ixjuNUrF$iCO$?(%5O&Dx0N0~`~je^E9M*|JAKoOcOTPMekCuoAgf{nlC)jl%bdTIe2sffj3w5^q?*LoUrV1q=jgND`c z1yn8v-T*-Eyj*zKZ3IhWN0v|e(1lh`viO`4>FHn~LvIJLme2=*be3D8kd>L^2^i@- z)zj`lo%g0en-Hm{K6vDh9Iq0vFZ>VM%2pe7T7q{0 z!j@pw!*byJMx-jm>tHEq_8xJl38SN&rQt9GcS`-J0^ul&yCc>nQSql$o@+` zS>R;wuaA5d>SM|`vOWf|s&O4ECHF4(5nvMa*`k0ksdw&>#_`{Sn1Go6+&gJwe+CL0 z6cg5a8z60v>)S^j`4x&P=z^xxCc@SgxuiClf6c1{)Da9vXW!q!~3ZqLk9RP_|~&NAG7RGkLUo3Aln=c zVj74cpk9_A0Uh=L_hoC&1&#zVC5RW*^YsScOKdoOh*yT9rQXg>NHQM`l5X()M@~Qw=C}h=9=OMfv^*&PHL7>J5#wr^3uAx39 z>r7`B_~9fVlaU&b#-0jkYGRH_?7u!%%s7Z*R%GNDa+=S_o}>vR(+Dxps(#6JrxVMH z4|2H7%F_y%9;4NXGkmr+`j{K`u`@x87hP!Xr=&@OKHF1fo$YBxUL3p5g*IMwhILom zdY%VnW$I@%xSu&3fHL`BDGf1ygiqpb2dO{uDHnq=&DIp<1`S$G)x5%@Xigl&^5Z=_ znChSKIfP$f3FL<=Fk=)e4QC|PK91U<+8?BEP$1dLzK=N zyX8Qvt`F$uu?eD9BS#Spf#s5N%OwQ0OEG(VH`ZW|9fNTMt$6m& zjDl^(1DJlYSJn&NTK{_$waVe>LqEhIa;1kNQT_T1pU0a3WL%k#ft;y`=@HLG?R_)U z|M#+DayV80DJcjKf2d3N;kSU9j@97JvFELjmbtCc<#|P@9p`)B-bT=>5pf5TH%@WN)9zNVRG@Am1L^vJkB7ws%dCDF{)idAfNExRI^7eB zP+wk}+V3HPw5C37dqx=6dnMm%xYQB*4UoCBpiTZ5a5gJtMPsUPO}6}D59!xj^wRF# zvwa@)5gzb;%;$k$#NChkf1TQYtu&l)rYu*@A#C}xRn!HRjDN~QJUof|;^y)UP^j_A zQ54v0o(J^9&vCg@0n3L^eshyDE|)I=3OlZT$#36Rp-r^NFk;_bIz&NBr~QpqkQWuQ zw!}prvm|*neI3k%!m{eHXTT*2lp2-e<0Z>{ zF77KefgJV;h}BYW=W!iBE<3LBse)>D{O2fb9ow0e^d2iuPbz90@zDh?v`O%d&x^SM zyCL24JH@Or?1r^Ej0y2?0j+=Pxk2rENaHfl+k0y_`o9Y-_s`!1X%%&!3MIvy-#x^b zrdA|5`wyV7?z*CpwwnP;b?;vxPj2-QI5lA;Jmy~y(8XB!RyN(cR3kMW&{p%Bfs|Gd z*s$DlX>$1Xp!BAyqzv1^M<_jpf;O*i0cvW(Yd>%71ZAA@Z3=uJo3#Wm&e))(B&W1R z&?!8S$9M4&Fj1xChFyJxN<1m}R7Yz7IH)elLJgMGv`X(Su#HyCnkHRzM>C&yY((^) zXGS{i3u02``T>tZ@F0~!mW;y!eX??4XAoQBHC_S2?6E&^NwocX$}**`J$VOGIv)UO zS+jw58<#l;`V@S3hmQqxg|?!iygq``eR;4?L=V(QUnJQbDAdh3$G|$oOgaRNK|?=i z+PVkfP`BK36l!#wSpjIgsaKAibd;xg!K%O9FwoQ5X=T=N(8dXFfZFk6$Ag5k2ZAa; z(I?=9BqddYp;^f3*Qw;_Ax7xwr7Tv$+EEX;3i4+ofK3wM-l4>Rd$!kd@yQnE7(xC$ zjj$;U#=2k6fHL{jC!%yDuHX&>GDUQs^T7EOoj>wS(6ArTeuK|}wDR=@d0Yq12QXPM z(&gp0;hxqjbaM9yXcHh_XIKJsiLw4-KodyN5Z7rUAMw$c>G8J>sH9W3r zM7TVnv)I5Waf~7K8d;_lutaJ*#7H!)5(J5GnM>wsIBbd_uOARtq=;3Jxl^u7LZbFP zdrGmEplK$3ITM7Ze$೟ijeo>0}xRnJNf|R4JFfvQn>!Mj$Guwf$l$b7R)0Zj0$ zSutVjrmpQ6K;wbol$8PNqW2ZE+mLW>FCjT=R@W=|!|I^#bgB&`KMsg0>TYg=hK*qa zmy_48gbwS+kuc^74g;?OGZ7{-IDls}F6SMu0k*ou@e;w7euW~X+V|@%S6%C4(A~MT zB5fytSbf|k(=F47-vDT}#xokz%M%J%Yeu6nxo9G!@x+|3cUt%W>CIqPl)gLRk<;SnEId+e~sAT2Yu4?uedKH;IjJr9M8fI?PV_e`Z4 zDP}+EF=r^${`lcj9xL4tl6fzHg(HhwF*zk@_adkXlr=2F>iGEp#t55)sD3@cK}e;d zi`?-F|5`KYj*1bT(ms0?&;-M}rd9^N1x>YiDo*~@u!*slaTcX(-}8u$m9YjXTMR<} z(V|RSLeTOry~u)x=_eixidDGf>e^+Ig2uRes#)Io!UHTL*-=hk31BsGPePFBnJ{^{WDiX!5U6a-)JyYJ*MVrdHNdwYYd&d^~gc?{tnFe zN7dO#=`b`-L5rxXemv=Uk$4T5aZ<2%5S~!Pl5(b!#E7p9H-a44*(GUtPeJ1(U#!AW zal#KiGkbzcdj8^(m>zZTG2=#rw`=s_g;O>Vhh|D*`MYc*VQa#AwZzr@3T-Uo+Kwdz z%R#5#eH=b~oRD=&n|u+aI|18qGZ0y(IMaAe!Q)e%f*UTA|0QUm;yONkS*Nq5+w4=?Nmt{t!S02pJyCcfNDpRQ^RWGWcrMveFgI@hXcdyzunpmJ?4RWCGDu?CMh>ja$rvw3@D>7M})awZb~XxmR4yIvvPLR5yk|ZdSnJu+4Bb zqSqSZa3zDc+Bcj<9OEh*b6@v~%UkDwS^1PEr=1Jw1=>r_9S&{fnA7?b3K)Z>H(D6u zF7P1F@-x8dba%oCKpo)gWa5PcE!UdJ=+Sa_1R7B=$H`I+TZVcdYzFX7Q4hLTmo-b7 z83VLNi`qDQQ5A?)+I{_%F4?vg$hsug`_(e04hR#|pcKdsE{fRQ!VT37ySxa%>U%tB(k}7ctRrb1!u$4Gk_3@_5 zp^a1hb}GguGHqn^hQ=k5kH6 z#m}6NIn06%B@lkD=YdCn!fuE8G$Ma0V0k(R)j;-s%;!?8A6JPhZ87++rq*_?A|^)N z&y1M^88$Z0%Hii6$keBOCRn5zWYqJ}#tKifxGD3C1wIO!{|Z^5G)uBzX?W%Jdm!zioT#6>hHyA| z*h|l}P|F_xt+se->wBD$wIE0rHR-XQpcSc;^AaVkNPL!9It6mh1|aKhdK;aOw%XA0 zqT-?Cxpc~rD-W4%1hb;_dk-LwD`FYy;!LH#wh@ZNv(Aut$L~H}6}=y(x7iG0qQO5S zA=m%y^Wl;=4=u;#?WX&dqOdTmlD&6;wyGp~E@?uU|C_f32gu|dJ&k)d-y>8y6iYVE z2*yVQd<^FoGqELf*uKT^t)?$pfms7`OZ2N(kyDm6+ov<;Up4j=@7+zgez<%B7&R~NaeH^Hz! z(loewOzI0_MHMsJBQx@?Le{)RoJx1<4{7Y{2PasY915W9vsF2aU}?a#w`6gQJ=*^7 z*-y4R-luKAYlQofg8<-*!wYG3g8x$&{5+E}7@(xoDK+5q9s*+AagyU6=DB$&kS-{R zAb%)ftnwilNuT7S>-tic3MZTlWO+uj?!!GZ6fvQpuKi@wX+Du22kN4_^%+2>dtB0+ ztCTUpOskIK^4Bn+P^I0o`O@S}AIS`dIHsKqWP-yfo3XFcIRG}z>g{oT;Z9k6`+Q(y zC`tL!YlKh7YL;{hF9a|tGx@QR)9K*mivW!)_5ah^uG7Up#+5te4x~-gXT{bKQ_g`3 zSSoH`QI|lbDHNts=XGwuRDmF3+=WKUze>>B+>9Pl?PJ>7P99JsT%K{gQ168xo#r*5 zR-yVV9~rnv>9F%UkE3q20IHFF{b6BcTi`Kw5Qp^Flvs*tQPD##MDBz<6vY%Ci*@E6zrbz4RJI!k)@}iJwos z2iPg_-pk-o05(!#8@Jb_ij7$K(Yiq1YapJ}^Q?5ajIfo1Qu5f7RF1WO)wSza4V$pg zc)TdaLt6iO6((I<2JBDJL5dovhf9?(S&yoI1wAr^i5 zgwFK}hQqFX5j&6<{)6Gr;HHGfZnJ#qveJGMLF1OCT1u8R(zQwYcgHFGPrY>}Nom-) z#(2x=#hR%A)>0{2>a{!l{|pB-`SN=j0ll&(zbRm??cO(n=N9(@8PB1wZ6GtIgN1Yb zp)zm=lqzgmsx)X~!d9wJ8vdh5CN*s}BIfYR|%pT06Cb&o?C4{CM9qI>lm5UWDCr)iK!6bt35{z&CdL0hRg zJfsWqj{;Vbe;ZLwe+JOx6IyYk$6O$706txM_B_JoBH&*XVd2I|F-!kJM*X_T%IAC@ zjBffii0u8m{hKXH@+~!6gQkP`~{PIi0uqyNEJeG8_RgqB1 zb<@^!k&oeQl7XE28i-Y;G>^iy!5Wa_1+j8i*y^Q(Dk=LA%qoe~4MwxROMEhXWiwn- z9=g;=;ge)oFHiXBKT%vaJ^B+6Yc#BPT;b5HM}O+0MN(0N$fZthAEUG4ZEfDmj$f)}}eQ-YlID|1(`C5-m)#h$-3XvuH^;b5*odj1Gx zO`eOZBXkc4?wT;iQK-6L43AC)&^u^4mV$~_ZbDqMb7*gI_&0sVZ~R;@4E@e#KSoTGsEt`djR_rBUH!D0LFVP%eig+U(kp^jgnIF zkB^;GOO9+-pmY@FkBiv7^lRGL@u8=!FKVA0*{?XYw1?m*!XNe{nKNo!M1m-KDwmaRd;&Z=+AsJuTC zFk^!^%~+r8;p4D~;$8+zt;HR|{*S7&4zRPh+BlFxaqU493M5zvkd238En3_up547S zyPH+^?k3sbS{#B?tdy4E?!lb^!GpWIgy8c1o_XFom+!AT=ehHaoH=7>&YW?=<(|fP zMkK}J>mODk=wh)YW89p)GJt!NJZ9^J6+2sjg%mf&cY~m4FVq2}SpD?G z>1zNXAQhze>mh`*GypueHUN2Z(6YojwNK^78-|S%l>2g_X6!!{(CeR)-o@29Mkx*w znoZT%74Z7Gw#%mrcY#=DVV+Woesw@*WHOIJ={^iZ4Kp^j9G?H=N|>xLCkvtcj3#l` z2*TzZXX5e6s7#J$#64&Xr1Ro*ii1+Nr(=O!1C*M{K{yJ-^#IKU22;2&VSNBM8h6^e zW7!RJgt82sV#Z%HkZzB}Mwq+8iXcO5M=-UPK`}Jk!o(bmJD) zzlV;Av$rw9+HSL!}0LWU-`Qpyq zAlzETI$EVW{~w?(P#oM0#2>)MAvG+a@&cjwdJpx>)R^6ulUUVV(>6uknz0Q6a zGaB(VmpnJeXK1FBJNR1azL#4JSQ%|{DTvv|JtCa4iFReIm}GZ$IS>VFr3Bk4 zOlryf+zMAfxB}eFjo~6)y2dt~B*iXQIx_1Ho_-VIe~~t(*Y>LbBpgY(bYVd;2DbOc zMOX7*AsOY#8iB7DVtLduFFoYPxBb4I(T9ip?hcx&`qPTpFT9xP)?S_y;YYods+zjOUj*Zq^A)F;U{@CX> zNJ#=OIp0n&PsPl7JaMnl0rr;rQ4FP zfw-ovY_ipH`2O3%ZVgK zL$cS|QlN7h-;3eg)n7rogsr&V5=#ev4J0k#gcn~DeQ&fMq%zV$vz$fMY_j<^9-uy#QlYGtk@6Q<5Oh8urCIf}I~Dh{MxM=t?Dp`^%}e<{MQSur81PsP%pRT;2F{BeSoTX0DR zqW`afz>k&wsrVa`FKyFcMVWa`^HV&MUq#^9nHo3E5$ z)XLCqC!QwLiQEB$atzeWt+Csx&@Lzn7;f9+!9x(4bA3E!Dr`QcC$M}W(Hg3>wFS656;?3YiY%lBEmZ8d6*NLA zKmFa?5-y~=4Fy3@EVCVmQ{(D5vhw3D0PdB#E{Gw7+W6_+ax`6vY>oT&04SutMeknV z&|z;NCr(nj)^X&1AVsCM+L|renN?5{lwzrIIYNauM50S{p~EViQfQ@fNIh`Tq8!n} zPt2(sGS*d(mn@k)eAft8tsd$FE>n~<($X=Zm+uBQWrVpxj$2 z_1mNzL*A2E2wXTBq$3a4fK@qJ)s3{F`w4OZE(J~;r`rR4&5OFQ?CZa*+X8t65o zZgmh46-+lxPd^yKZIDg|;&K6Q@bL^%#TIM$A)xdIG$8vb=u8TUh_?>|a9Vr3v-%;eVluE+m^T#t6y4 zwwL73xcICQX_09l9$a`}f-RK>yR`&5L)PNyiOC%WQ3)9nj=YR8lEmGTrr1D)xzFW= zI7ei`S@aK=LzzV=U53SDqDh6NdAy@AvW4HtH?J@(H<6>SBJ7&!`VK|B;c5`G#6GU* z9fPj}%LxyUmv4l2DTXnMuXhuqtP;8Rok=q@{y=crgov_(OFJL=^kj3Hxo%u)%&^M})4ZVHw>K**+w7LF??s?ri zYXC;>nC0IE@VmxjL31yNi|9A~@SlAjP%$3ZGaGh0_h+nL_u=Fs^F?9iOu9C<>jOC{ zRzR?zFMfPDLmj?~-Pv$H|&a*)mUgt<&i&3O3~jR0vG9<47zm_Nx#B(8c1K)h#`eOzI)2K`uxCEA%U zgE)PLZRw^;G3He;w@UTyK_!k7=ybbyPO3Ae)c|%SH1cAv5knF{DoWMa-0r z`V5N7Vq*7F5p3#>{l5Tp*4$n1X-?O!&JbA58vB^Vzk+tlOVyl)5(;pPU`K0Cic4dWfcYUC9`WOy6@iM?qMod0KLYxd zP!|^8Vx957m2xZ}J10HbS}TKA%c*0G?FE=4N$=hfE3E=TEt|VzO9hJ-q6u}mu@d_W z^cQV?EHrqRdmxC*nBGldx>;;cj?~qOlQ6tptO`;cNtg@Osa9Pr$6|igOK-cbnGsH# zRj_A+M=Crs@yEqq)- zew=> z+X0xue8Jt!dTid^{%9U$TE57SHck`%)_-WjAgoFxQkty3R!w z-WkL-9Xm9x+9gvRz448xqQ}uEQK;kbdbf;e{hF9@r!X_aZwP2)I$-x4k=iD|5wY%G zU@juAbep1Ev@4R%2HdHC6pC=s4`Z&q37bFid4^5BfU3yQPswTieH|Zxk8j2imdMG` z)XEJIh1AMmX#>$nDiwWGLE^Cl?28K;b36uqv@GMH%@fWPuATrzad=xXerhFbdBmqj z{H?wCtCM$S6gF!ydXuTze>!Ujml7C2pA_N(;S~lSbzPaBz)~#k*&n(P5{*N}s(054 za4|eF^LJJ{@#P%n_JF!9vg=%GBBUFOONBbrY>c-jffZvDBW80PI0b-eCzap32jqX# z(mBQ+1mJpet8NcbRne45@lS{NV@_>2$2=SYUbak$-xMsm2R=MDT zYH{D0&@P6Z|5W0LvwEXl#8kl$$*{TW^mmrh&j?t@mQTOY`5!Ni3S4%RpRgy6N!P=yGVv zAm`-mSLE0g-Rr^*`l^h``#)uVNJ!BE&_JmJta>$&<%9dR)m`&Qh$tARQFRm1u7cFv+z0;q^*IL5PqxLF8=+~A5o2Prn+TdsJfX^cjvWNJJUr@+ zlvQH=n}N(~E~&5=r_ui|+X1LAYN$A5dXA6KB)%Pe2auCwDC~>ZL_10L>1iCB_f8;}o1Ry0 zl0(wR?*=1PDIoW_m$3P)(UX%&HOOc8ffiGN&bQMsfBpS{MJsV4sy@*t$dwk4r#=Ab z@?cff7Slx+OyHP7wU2lX z)E$s9beCCBh2ar}Pph|d=yBLmJLVN@Ws(7~_PzCdQs zyS)bBv}r_aS5VKTn9ID*AFc*^fx37|wDazM%-0qu4I- zES@mO*JG9|z6I&@xjfh!i@uXTLCvfkL@C~`VT}K+JVd}dQ}5<@seyDn(k;v~P#kc? zY3~EUV^*JW=Ldw%&91H(_8}BK9k192e@w7i#}T7q=%>&wBDV~p>odn3IwI~;u&696 z_tq~UNnLhgzkW$LCxS`KSJ1AzY=IcPD$(@~nA$)m825Y&K>@h&xc+wp-TGSY*Q38c zdpU{vFaVmxJ4Ko zWxaL4`~-`#ac^J4!BjyWy%;gtYy|PZLV(5Cs>?fU2LYV7yo$!*HJTcXijPFQsn8H5 zeG9`%+ftx3O`4>7m(EF`^+1ch3^Y>0dp&XUviUz=;+Yg@F9&Ub;#eWR5a}}3%hO9T zmbjdA%vg60P<3%GFJ9^VOa z))TmoK5KPIbzEh!{~8cQSLux>)`D`Yp%h1XPnOFLb#xR%!();N@^4^NTda;S`A_MK zUq%otn9G55U&KiN#|7${3YyRGbxbTi2HG{J1E5;bp!=;rm%6(w1q_?qO@Lf^U6w+K zcit4ls~4W&Nq)4`^Flugrw%PsRNmp6J&q9OR+9sjV?=r#Ep?Pm1EiY2QLuAkHcA8eTeT16UiI3Y zbK3a3@*MZ?-N1?hsW&h2kDQ8xl~FA26Xd$VMph_tn7{cRfMyXVY7AiU=3XE!R!c`S z&h3~^m}@WD8`w3(j-DFsun&NHQ}J9L+nLyMUr$`!$vcTda|&6lHwMngA2 zXM)XMa<~153Q#dr>Jdnt^{v+fmPV4cn5wWdYwD%_V%v!TZUv4fBr9z~d=udR4evCB z{xMS~X~szdw~1Yjg2wo0D>;A#vkQ%o*<`)S1}odqbB&_ zkQ~p(Y~}dhp+K4Q>=ds!0-)G@P!vz*2BC*+iy(99n% z5^mrjaKOm~i)NRf!I)>N|EqMxISM+R_7*H~@I7@Z2yMvw8*#P5F0I~YZaB?aMqUV1@Iv<#Dsib0 z>M(pPXHq$SBi&VpFWX)QWY)HIvOQh*@=Rj~Scw+V&LSPm(wzrf0pvn!UKkR$2yh1& zv0jY1GJnC|8G9%!%&r1*ChQUG;tkOhi3QrLR}*x*B!zcca}2u%&{Dw@wcZpid2LQZ z%bfT^v}}*FIp!jzY{1=?d_OaCBQz!~h*o_J%*nHpVSDyCw7UqNS)%I+NOO&)D%X_d ztKma|P7Ojq1U^zXu}Ba>c@3qc&|AXESgpOw7lg^U zp&@Qouo&g^4l3Mzt@xKq!tW{Q7LvNhR^A)U&R{yf!p~AEzMTVT_UKfsK0f#!fc!dX zmZN{jU(lv)C0*{1w*{L2*k_mHkH12@B+2}aMECg(sAx`Dvf$jSS4)%a?xItmImd2A zkM=$+#N%Tzut=`@%>(M>QFoPEQ1oshsOW6Ixy<71+kSxNX*vqjw@>{6%tb|&-xdqa zpJ`SWt#Qu+nbvDQvDbpo7RuI1vCKkHMJqLPpI9jlUKkAS=vAwxMF`Ux!|USo#R+<_ zDO`W29oKDw)pqoU#}pQSPU##qrirGs zTywc<06}L`nple2BB_&>F`KMJ*vYfxW`)szWdIjEc_`JIf1y=!q#iV~tucEbK<l;U8jWFHQ=BQ+U zIj&@DzKtc<1!90o)mnEbgfl`}8>iPnx;?NYX=>(TJ_Rnt^}?MreH)usr1#3<`BRZ! zi8Kd%(=HYq1p>=3eWyd5Evn#9588H|{EP)r)3J(LY(2sf-qG2rG-zJ2*wP1WSbekNTIj}WG zYzOVcW`2)| zo^d%glU%yawOT!p8In#P+Tx5x5Z6LtD1|X?Ij9Wk`l1`-7BdUx+iMu(uBwX@%keo2 zW?($1Vu&orSy+7#Khz?@hGZ2utBFW(Ni)gm%??kYHe*@|n?pQ%#85lA&3|(K9$T~% zcKvkx5ly0t)|KHc?>csZ!iae3{`oUp-F3wM-Ov&N)-hizT+ABse#=^F@g5*a z*YFsrVCs>KS))W0xp4rDfb1-!D*qkPERuBF*%>|9>n~wSN7PM(D9Wv;(0&kI3|S*$ zU~?cScp z?2f4KnA+-G+e zyNE2hB@^tk=~x97k8ic9Pir*h*L83Hh#S?ifmr)Ke>7@r zyr!UeElUBN(4TxiklU!Etb>0T*70DDhJ|R-?A}p>CUwgfeJS>Qh*+)!izX6Nzq?qV zMAVLBbZ#mE3Z*!!7SFFwX$-sRcVMAoXNYY>wj1oK>{8Vdwn@=t5OJXi|mpTx(QQvy}-=2m9yD4NPh> z03I|CVG2pttXHrY?Jy;X2SgW5H<=A>GKX074}j*ypt|^EUMNyhb>}GTewlh|{3w92 zk;Nb0YUYD*V%bwg93#*TkRBQ3IADRC2fpSwHPJ0g$KuIvg84(LEk^l8o*yVo*aCC*wU;)k(U0`e=5*D>gWFJq^ zwcuh2hRwP~5laEO785WyVsW?EpFmt9ox=@@dzJ<;*GQ#dStz)stM22L%TalEjG^LY z0ZvWt%poSHEe~Qb(IXa;)A?7(kz|O%HRnf#c<@gSdwqOfiG^1LcTsc_!`=EC(12C} zH@*VQQl1~>fCCBX`7;>hW!cT*=KGbPT!3U7&3N(5%0LApbPS(tz|USK$Ezf#oy~E} zKp=R>q0{z*9OUTwc?B(L6Lk9jrfAohi+H{8v~4vISj56LUQ^h4p?;r)Y8Mu7ygHyu zjeSU3Va0F4%w-NdHe3UWMrT@UjRQou3OHirc5x-{734y20@55Wi>9c(_0cgT$CkqZ zO@KEFA~ktvjyWQ#a!a@1^7K0Xc+7~Hrl9}CnhDeHctS`apggnM5nHbd;Ks`8F*WPW z!d*&@)av=42(m0PAH>+9kWO5@V7z}oh!bbvMEa)<18}`Ktm=uIMVnJxz?B6EJG)vO z{0z0m=n=&6@S)>vjZQSelB04;aD3tGzoNx z)5#l70mt}@Qg=*JP;!>$y1`?iimG($Tzaek^CS5RrZbe+19I)SmDLod`~@08;a!Z_ zW&QjgT8g@OPz{#w;pK>9VVQSYaQCrn_`$9!U*e+M0ig z0gqcrZ6e_iO~PpSswfw=t&?7`#kSC9ZiPog=*{m5p*eUIWSQ;qpR5PvYH@iXPP?0T z@7JkCz@J2gwa^YbEP{jJ#s0&hqg-Ilh~<6i1B%GE>tog8f;X*8m2;aJ?3vDr^R$vSD)irV+pfLu@pq zsqN(o0rI#=i9hJ5x~>zyl%0gM89O&QjGb;gFoCeMOP6}p`JNSK-s;fo6EID;T62B2_cIiPjp+{WxwW{B_>WAo^rU%$7@jGK|VSUA(r{ao>5(8*~3B1G2X|n#2KQg{g|2%)Zh~cI?1}RC`B7i%-;^nanvhb=do~C~!F2CF z1s&hUB%JFCq8-|q5Pv%-e~ax4*6wk~xgd0u^uS)+d>)i*g8nGBKOYj7u{InT%Uu9L zI(h~9fqxS=Gf;c9#?=>Rx}nU-_~U<}U0dI3962pV!0e?GGcPx-rL%0vUbzCAoYGu( z+*J^+gzs?iR^c^3mNCZOG*iop`}J#qs}h3IeI-u34#YXr1q9p(K+5!o=D6x+2-i~_^$2}o2XM(98Rc=# zHg3*I%HOyX)YU8Xb+E(ii_PxIkS_Ua$ZZ$osxdXlCT0D5b0j^1qDaf$mm}dVLweo$ z9|W;jRi6;RqV%EsC2Okm5Xwc5<}aI>s7Y(=3=r3JLNAW4_Ymy@a0;zm-vvU4B+_{WL1UbtF>_6j+C-dhx_hztP z`lyqkOrAq82z}1z5 zusB_`Mo1R&OTR$a0+bXFcJ&pbyt(YmJ{Z-?`N5nY_g15;KYIZ>|V3gycH~7a@(jb25n~<)<#7>>FUGp{oOkma2 z#`!aHv(7spWY^UkYrdCb;{<}D9Pt4_ZJy%ZRcGw@p;N%`4DX{Ug_ni64wONU6USGA zNOV$5jQlvK;oGt_(e|H!xmXq4FZRTMPXWx@x zX}_Gz1Vr!(k;OWL8`muRsqZR(05k`1*e2oLPe|2jofCS72ZbiuL~Eg5dw?VR8F{`Cl2R=yJuNMS)x(ahD=C2=NHO zu3QIuNqh$`R{OR%KD=0v8I=|^tVKD59=#MO@}M^;7#SX2I)CTa>DVDG=NR&+gdNRC z%R{=sxwDGp%L@7LIxJ=z20&7NX2a1d=6{B9zjJSq{4;WRoT8waB|poZ@y1F3Rlm`^ z6U1S0gCC`SZDsy(cAV=r$2+3^DYux=Vb!eyP)u*d92Hv&BP(1ZEjNf@A+j8D>SST? zYWa&wSr@;NhP8wtkaY;;)HU+|v|L}3r{xBNx!cH;{xFd)Rg?V9KPlSn!T>Kr?PJ!; zF&gDf;kO~sMU`>2fyKZ&If5?c$B=cQ3xnC@09IxjX@B^!{&C`Z7M^dpSgOdSDYK4{+40CmEs3?ZPr`%@oZjs~(cY0N5|7R2piYss-Q zVvX^$5IBt4#wC9t=$^yYP#0wIg168H`3ss;KJQi*;%FF@^G1>;(u)MTRGO}({%GG2 z$gJh1z1G<3Z<%h8)yf|>`Q0b7IWNchg8WH3I{H?$877CNH2I;M<&WACbeXKYEr=H9 zOf@#yf-toiJ1Tk=EV7q}`01jZ7GsM(T<-5V29NiZ;gqGKlk-xL=q*U~<< zxP`tQ7HqmNkSoKrQf79}J>mv|{*c#DQ8w&74!}AuyD}`C5)ZG>Uo+}4i%1y93#2(< zakMB*jWIjfrGcO;Ak_;Oa_5vlJc>h2Ekn6*tS`C7(h-Bl1GzmalUjPPOh`TW5Wy}p zx_IAUi&4#hE^|ZT=l2r;%*6Ecl8W|^Rxm^Y+vN1<`sJei5t7Ub=>2v8H`wr@L*j%E zNSC>Z{)+7H-UZ^k8Mc#rRN`%c)Dm}YPxcUWigLn+{qH0IxWRqp*tiI5RC!!sSDuHj zm!AUVd|8Lb8wWya`J-X+ClQ4-a3vAL5Il$+3??2k#a^qh|EK-1RHh#-b|{#c$ucrM z0rl4-fLs?wN}fcBTaR=iGVnO{pM(n;M^(S%asG=fMr^FG|KemQP88{+Thg<_xFbFu z$d%?kC)%<}Cjz)gJST2l^_xH;HsoGTbV~4+5xZBQK|a zb_!uv0dZqn9+#d9BF=Zlk*7JL?n4bei?F*sh6r6J zxwLa1?iE%H2e>A=6^at^ zJVNYyTmHPUk1H(kvH;gXTQ&M2jkd;gK$i;3k`~khlZCic+%<$5OWy@Rze2?_Caw|T zHlxvb-lIm>jjiMVyZK*zj1}Jt_Yij3FgsZHUMOd;9>+d|?5`A9496NqlU_5v5o*q% z7l?K4&*`w^eDnb*mmnPk&VCfSnl47g>0;x9GXOpI7DoV9xNr0rIK^wh(dRn~BW}D% zwA$kYUB%HcQ-l*_R_bKvU~KqBpxam0cxYbZgqc8v)Nq>xm#k0b4~tCMT750hvBuZS zvDK8P0L+!1mN@tsC|9)${r}&eg`|A-lyAdXgqn!&MA2qlS4;fxAyjp-#uk~uj(tlU{1LF5ucAxLNz8v1XgSltLoa(o zHnJ;zlB2SDYwMFrHU@kK=z3Y_S&4yy%u`-@X8n7U02tGS4a%1UDHDzH`+GUOWK6of7OAa_Sls-RzaKd|W+-S*GJ?}u7ySYN zE3~ql@oP>DSLQ4?)tnpu21p4hU`zaIp21aSReF=P%nTte0P9L!-lijN-4Du3@sFROudA9LzlD+#r4y z>Bbx{BUxNtE&*a`((813C}LMZmST?H`!5Ama5Oz3v6t7UwUvvFNU*EJch}o+>X}Zz8$|_(qlQvgzmx(us^#Wlf-dzj8RUXE&^{{mzsq&QGXdOz>S*ImeTih(fA2TvC_&;0+;K7H3F3i`F z2V80`KLXfk_MnJ7Z)E8p3ZQUB3(Zjb)X=8s}_Np z#AWLffnv2b$Cev8$a~AgaaH_ZYt= zVM*1P7`hcf_gi+P9g{e`EXOo~#6^!BqqcUqwR2(&-JY;>sbeoWYX?Z`j<4wEn7AWE zktGTa^|?Vi0Xc1%w#tkmt)xpg-Xmwg!ep>y_tbZYpu(iL@f(83(08CA(a zdqKKTEMKcxKD&30BoD7E?~^0Rr59aFy8Jo&f|?l>+5RC2kL?F!iA#NRk#W$p0n&MT zeU6P13d_uLd??7}$BQVh97cB|h?&8DVZ{3f4`Fpiq99N@J`Zve!6KenCXFGFUCQcVpL}F8cmL=|- z1Oy+)4vU#ZuoLfnlR-T!uq{G`#05{<<%lW36p}4m8n<2-T};LZK}DB}OEKes{IL$G z<@-iQz;_=4T=X%HNVhl)k_yU0-hm3cT#RXQa}@&)2XdWZs2-K+65>cr_1GXYQ5+)3 zOk?KfG_e*19RvHDN}O;cQPfw5#ViGtBlpr?RM_%c-9*IC!dyT!!m;AfkWL(@BHX(= zHh)qzA&bev9H*p*f_@e4GO~0|OT*?9fZWp3Voi>Jo|pliu)|N$uEludR87gDi#~Ou;d&m6GaoKqtqVFl_>sJR^UG4O2WX5@wg1V}-K_S~Tq0 zGo5H%cs3}#h&|ZQ_(4QbA5M34zvUM}6i)`%)6Q|2#b?~Cu!Y=J;#ezi*~)amplFH{BNa`MeNttK*QV~S>`Qs9Yof+aN5k9&=hsZ z^}y~{Qm28OTtnz*=+xptA(A-kMp5%h+jBE2$oD#fceg1F4s zOZCO-H$xYzEG=tg&QveVanSql%86(vEc<)C2Xx3SK(qkHHnI7wge^-r43X@WI82@@ z)CucIOouUXzA(PRTs%fhCv0Z7qlP&74oIwm2gO=<7Lm{rd{Tt#@t3geUEr=u(o1y2 zmqJ`F3=h#<_QgN$0kXJMy)af{dJXJsz`qcNIxNNo`r~rPgyekUg?oW47hN@%g_T(0 zK5*yEk#IfL|KULpR}NM)fWo!49|BY5m|S0fgfMNS-35LWW|)$us4qSeMT=pQa{prl zU0T_S@Z=tyX_LnRi`-ZTkn(&_l|7%0lv zv!C%4R8{Qt-Z!b4JMdlH6n}mL(iP>_h@U!|_Ga<<@EG+D z!KyeRO^o%O@BR+bQl%2lz6a#`vPbWWYd?f`eJlIN>!Oknw8y<45q8~WoVwn}ko5nw zk@@0N2p5GN1!i<~?7O~l1hhdNJX{0e!qD8ESRutfW&@Ek`e7Xp#ZjUiWfJBklAVjb z0kS}8{J;|d*GOXypjnFN4~DYqzXfpS4LtJLtV{Dx2y&5H)TQK!~t01*aPg)%F69hc6$J!r6S-^Q0j*Aa5>t~Rn(a}Zp z#oINYaf;q(xae166kNxeD|Vb~txQYF*(%z_$GkLc&wB+q8$Dh?q|byn8!pyR{%>nQ zQpLwoa}$FKjAdgu4?!~tgBAKFEzr{sOhY)&y~nYC$Pw_-i**3P`1pKaE@rtMo6cFH zb5+x|(Ld&|FgW3YD5{uVVTFn*JzmTa3^VNE^8;J5+8Ux~0VpS(cI3~CM$+VKV$6d1 zKO8gQi(WMUv8s$`79w0spd3Fie4|jfcriei3p-_L>sMSnCxHf`Csq|rRapv*jL{-I z8c6NTE7KCTmL))|^35Z)OF}v$od7EuPpt_w7pt#i#P&;NOnwOZl-9<70&{kw;x`d) z%r3?NvFNh^a?@dCNzWX#G=OV>F^djAeil+lFl$oX>YryBAexhT@{ncozg^m@A!NHP z59G4R%Pl+oO8l|Ezvyj<%@uSGr7{`_Znq5paUqi9fOuYjv#ctBeq1p_n1iOpUj89?vQm%}<{FReFyC7>N0NgVuF1!Q)pBad$*lJj9YW0m zty;=)VGU?+;lsE_fb%OnxW+fD13ABRD9G978X#0&zIkH#H6dIpEVtqzk)sp z84Tct(&<52mW;dn4rA?=-tX8~7>VjpRU8fERAjX|goj=MT!{3(PhWg2#QZ{G)GO^b zBk!F1_Ab8&^n40~(vA1D5g0lQ+zc3x24CX+>u?}fiLEw0 zbb}E9nzcs9Ulnw841TeEV#PWNsNk1u<9Q5Qh*>1pD7%k_ba~RzfegmuLV+$ghSsc9 z&_3-n1`L)oqkoGbf5{Q5PTynE^}#%>Fl^$+dV@@J(lRzattdP^b5Xl5$@GA~1CgtiVC!v7xL}yChazbE3Up4G0gu7z zKc}IV*Hk%EkfU%6#G8xTLDO!${MfmDj*UiGSEl0Goq;@J<6d~#U7%bwR8$&Ne-`4p zbBf3lPLM7XD|_s5wBUbEpug&F?(1o8k5hLCsS1c*59^6D1W`5ZPd`)GAC$Ce?uUUFO$ZGHHWkSDJfIpUUjHHho2U65w1osE>2N+Yqfa~h>2kmTR zdb?sogFi^-PVt2(GQ{3bwioj?Lb{6SkyiE#yjn0G&=u0zpE}6ff{@cTY3q+`f~fX) zZi;CQd~pkyi#5JI)}N45V9AP3sxLMdWScRj)n1vjVuMT4v-Jnu7&{`yw&u_Lzz+7$ zZ5b-sAm$h3>^VJ<_eTuU+riAoMs~PUJ0S}Nj1GxuO-&F#2__boo22xaxPSf#v5j{{ zR>h46Bo^w1c4IVRkJ%j8R{$t!OGDK45X`b86SOHp9Gi1jGW%{KfJd+7UQGToD)H!~ zjFveFW2lti{bzDc4u|Tx|JSb%#FZohs~0%6j<8(7(Q)IHOk>~C84rs#6WGtifP*rr zXH#z%=^|@RVLF`d$(}FRO*yfGi3C)^?OyIyC%dOZ!HNNdf-!O(4xj;q z)0syUb}2b=D#t-b_&+h}vLgvQG3}K(nCF`IQDEj1^AV1hlQR1lp)T!shB8*boBb2S zrKLq>c9~WLj|nDg42EOqF%HW``|}Dr+2lHe8JCgn!DB&PINb@2=R{j7(W$fV6$i9! z*z-94a37Ed0kMCP@-MVMt814{9u z0QWrE*!Y;M-$^+M)R-ew%-5O0*h2DTe%9N;6*(h9BF!85Ur+}@WHDc zX1I=(t|9E4@pxE{o34j;9d!3fIzb_P$xbm^Zy@Z1>roFhQId}YxtVY;g-)8x7QG3~ ziE{O-GaeP~&$#d@Rc3rF$Q3}#%38a8O9n8A#ZmPLLaTrITrcrbs;Xd*MyJs!%Ravp(s2*NjICYXy*-`yB< zKM7eBK`xeQsj?t=AZIJ{JeB`uNh}W!u}KZW{S%XKC5{(Vj8S?=i>;5W^}wu;nwV~+j>h`>nEVX1D?$gt!cI%w)t&`(jd3CGwG<@MHU-^{TJ+s{f=-T`x-a$RicIpy8_>BN?9Jydwodmt_<+uqK2?0smu zeP3@heL&EK;aCaQpC!NbAYj1}yiGEy*#N7+|t@-}<2sR(hQTd3^G*qSS zum~DLJHR(SA?!kzuyWJIm`beqIbcn7R=5=Gj_tnihn(od%&!Pj4c4gIzQxO;J^0`N zezUI~vyGAQN`+ksooj0{sUhR_mU?&05mFsViknVYTYZzioY1R>U8Apt=rjVk8$E|O zW+&$6yf!M@a>(v4juz=QuU=+}Pldq@8GyyeZ}a!;0EWhPh@Ba?U`8A>T{-s8iSgo$`y;Ijnq4MwZ1GJ0N+1gf*-i}wYe*)8(#2Vu7KNGgZ zCr^%i)rD2gNxuRkBV&fgPYSxen1+mx<$r^A1M*aY%r!7hXcg#eD!u7#FK$rJJ*0|h zZQ81(^C`kCP3(&&%CGl`d2%$rltJs9_XjZNq0J_)ieUT-^X7flXCnC7b8TRy$@=gKbc3{xy zD6#aCj%u>tqPJjk;cF2`3UsAeWen#@XlSxymB1{$3 z2~1)&Nnc<1&mb<8o?C2)vH(ilK>M9m`;#*XP6Bpcg+I++Y-Oy$Uv^*(!|?+{a$9UT z2+XzdZ9Cp26I93~!$P`9caSjGkdtHP?AU%)5SZ85DlP455QsfX;=0xI*K!`gP=!W9 zn2TOwq{eV!4FHQ9CS6QRRIwE22z4~tl9^`FH33{WJQ*cnK2VTHydrrY0X7fLQ5jv* zJ&8kp2lbTMhTt8(2I=HSnI5nfA6A88*qB&)2z0Kg4tc2 z;)xN^5?P&3o;ec2MV0Xel_`$`anmIok^0O`Va^060HoP78o*pl>Sj`!EvTrp9{Q=o z(PKbdEAFd}8e5|}a_fx)wt^}OSZ23Oz0VR^5&X^m|~Q_fL!(O&xt!Qt7TZTYL{PWDn)i50iYpO^G(+QXs? zMr+LyF9>jJ59dv(8RH+q%83?kBI9F0MZd2OnLq9hX7=Jy zsByLJx&1Px7Y}tJ7Pks>dX)~D$d%%)`W&sqsE~}*{?q_wj!a5s z^y>+5IU6Wwtk4KeXJ)+tyF~a;870OYC1^P&ciyp&`^E6XVKM2f8qHc@-9-5=baYDBq5PkZH zjNvz@y-Qk)`;W|@F(}7%M?ty*+5>uy`$V97FrGALi^}7fj+G8P8pN%x^Z1x9n)rV2 z?PpQuymYlZFnTP2`H>!YMvON;}R_s0_mn-91}il0w}EHa~PdNiy5NkC2rKiT`ofK#AdK$aq!IR%*ONR{3rO?tWk zr{<)1O&WK+eX-hU8N%v3Z8yfA4&=5$XOPaeVq0P6YN=kn<26}y3v@2kX9Q%Lccx&M zMpr+EMBf?tJNggg4| zB>t=OoeUe-So1={mMD&$j}S#m)=!C36r}Z#a$RkBOQh>0ab$V$pA0l4oxuzNZf^Ny z-0XjlE`3)or<$Fy@+BZHF^ju~I9)VZV*o~%xKfleKtYa_EjA56-A|k14@6XJJ8VSE zb(R0bysRhU8VJ}TN7koa>py6*5pjhGFETNx$MhRxKi&hq&e2(K#j-az+{CBfDQwx` zecH4`Yl;PL1f&$n0DF!ISCLEqSR+BYimdZGI8<&FVqw6rg*`4WpWg~Z;X29fV}+fA z9VuxD`O|d3B2m_w!$dk!y&ggjyzUOC(2YxyrxmW&D0%3UwxoP#M%%GTz~)j;4Vr~I zB~~z+7C#W8valE1^De>`3@-nnc!%Ye+zsaPRd{vMy7!arseuSQ<{U`=}Rp=0xh{YR(#D{Uo$ zB$z+qkVgo+c&v6i5MO4=+XND!2g|MAM+v*eXjf!-+J6R!bMrOwIm zjxhHDF6H-s9Et>yQ5<~`u8JLZY)_a8?c{jep5^*00-T&*RwU0~g%zc+l4kBFYe*^i z@Lf%a6Jqbjz1#-{(DN9|o>$O9)Z5CrE5k$or-3XZn5D#YkG<5po zr@Rg3GH`#8Lx5>DKucyj;BZjM#J=?G z{1GdM&!uQzX7;$>}ocx4z8bT?dn%0#^k@w4VeoxGQJk=Z14)MYk$Yj146p# z5KiOibODrVB4f%u3cCzaeScH@HHqPLpML!1>^av;n=T4<$h@Ggu)NAi@-LkasLJhj zF6qAd#|%``-L;_$0Jumv}oI83Bt}57c*1X-;?V^f!5k^Pan;TOvMnOQ)-b}f=(>UI4}k+ zMP%oOp`f&t#|Uv5F`{UWpKCM*5~Jej;D5IgiO0mHA?ZQM_~x@^@_$^%i^Z4CNhSAz zYl=2Ma0J6?-Jg~N`2Wj63B|t4=XjGiXQT`@07v#*51)GlVwiC}^o0L)nxsKk(S13j zQC(&Uf>$_ikEQ$n{%?HD^u_*yTvbkInq#RIp$o-_UTS{$@yv?~=~f8Cs}?u9hQdjg6vi6?W0Lb;;yPMA)9liR)l zb^PI)G4QYkY2w?>1QspM;M63tdJe*oGWI&jqV$Mi;7CI1Wd{z2aEDFqO*&A^R^p|R zpe}l;6V>D?qo5g{rc8;AMiW#gWIctU{d!OqOIE;oNNsZ==2Szn${F<+0Q0+^M`rax z*m@i0IFn!mM%SCyTY{M{ z%+rX3EEMkC3J~eidDV{F5Uw`L(6MoZXiK)vf;t&>wGDf0TYsYS!v5P6c8!v!wb)L8 zD~)|UMzc{C;#4Pe(hm+6U2qj`RmHZn7wrh@yozJPG&n!B6R?|Y0&BfpcZQ^SSV!Ps zeOCxqn{)SM`cStUkgJ$RXHM$cV!_=3T}Cb)Gq{WrP>fETnz5z3bFcgzL$mf=@N43} zfV3v2o4TpBACya?rQYaqkWN&$jJaYgz{1Ij3><|;DJcc9E_k;92M z>w#Sw6iw;%mB#!rDtBg1+WbEyFr=&(6Voz5n9!uxo7NZ);d0bt;9ic;1<;^c()DX1 zY!0!rOpcqI1vxia7r~y#gz$kB0-r1Fwqk6@b9FPcE5*g8w?q|uqf?RlYAqnnU?StR zjxF{R#3y)K#HeykEkX~MWaNV3z)m8#jief3%44*=QlOKjDQT1VM^{cdZ9|!Pnrjed z;=DY-gBklfH@r=8h2}F+FbUVHvlSE@&@5cvO}Hx8!$!rcqMex@SYm$Iu>#`ek`WY} zueNxv2h5zMmoYZ?_GW-r&oEwmUVv-Qs|oU58%s?Ds?PD8O|$-E70@m;w|p9HG>#6K z1Y9lG&=Ju(8QLPoIz3(%2`5Jmk>9g7MY+%np)#tutS=*oY&{nvj-3MH0?DO_M)WrY zxfX1**kBGh0Dx4JsZ|^+%70_dPoj4n2;efYAsrvP9Rv*@wdv@phH=Djioz}qD_?E- zcR3uyveMNU4~z2ucqQuQ$Y=K>K%5m*k~VQi0;q|o>zPO8KXF{cGVH$s%nLeqd@Iu9 zwSA>(bfA&mKH6zu`4s2mzw3x$WAcS%D z$zVmlgcCaQSm=}t>C$;Sda`2$nPGa|9W)K1 zNS;LlSCKCTa9WMJy3!GUy&S|6v_C^<%lKsS-|q@g%O=I>;3lzm#=XL6uKJ0w$(00M zQXFe>K3|R#1UWlqGp=bLcU1;3TxZlc{2G8_4yTKy6NjPK=7d-Za0_sJv|X3K<6MFo z?|D4{wPJb9yJR=yM06d7ajQGpZv>-Y*qFA>-IU{3nq?#dBfb>oHmv9H z;GJ9ZH+WDXj}@i^m@Qg@Hn2*;-SJSNW*^-}@xHtxBT0!@iH+|DDdvK9nZ!#)+9J>& zcAz_CZhi0`5Vsp!|8jgHIyXGax1QMM0U#>PC4|xO!h;YVitxxV?ja~=B0m|(J7e!z z4?Dgb7{wPN%xU{crqahflE3024r>{Xq~cD2mdwJJEkk^pVdsG5yXJM~((@iCZXt>- zl@6Bb{}F;Ha_utaDC{iLjeDIjE&T+TtEHWliusZts?}VE@w?1~a69Ngea9yuNe*ui zar;va>jYtqrwO}FwW#Fkser23!{8auL6fLf$8pm05SCuNa<^2lV@osg9P6Uv(Y zT3-M_YB&`?Q9%;W4l(+_m?KIL-5URU$^UZYk2kYLST-72lCi^6*Lqf{Ymc2;9x_$^m#u4aW?EPSdPhT{-{p@%^G=TL-5Mc?=vuG$1qfh3q%(m zqDSD0AoaNSbHIYrzQHhsS_hO+BbwPQO&9Sn&$wxklH7vb=_&g_@I%4?1jE1>;k6MTolEiTi%d5X-vc zvAp58oC5wx>m%k~r*9(%cohigG#Sb1rSa z_7!G^VHC|?f2sf%yo0L8qoSP+yNME$2?O^5^8&gB*@d*Q8Ial5O7rEkc!iJtHBNvl z%A<7TSBhH%!N&&F({C#5^2nzGIwh|8*3!U96K*E{{6``o7WkOrU1kwZkP(1n~Yp2(=Lku8bt3ed(RImPI^-MEc*T=a?mSip=*nPb zp}Vu&49j1YdOPC;kySa<=?HVydBT(lSDSi=6L#G(tQO(oGdB0cog@4w3%I)YRD|=# z8C64Py2WzHNI-LdshKvsvo*i;=qiWshi&fH!UNq%~}GnO6=;ryh$gRQVZPN}_<17OXZ(>DYwItJ74 z2O^z0LjsO5MLTor^WoI;8|6=!!(*aI7^kJuYYMwuXgSiFt7l^%bI9JJHEo?DkP6q0 z4L-JW6@>>Zg^Qupe$|P6U;+*^-ha)38xBn z;u?_{w{O@7#3iN=;I(syeF0n~+Y?t}h9G~14wMcayN?5Lmc{tm!*#s;At!k0cx{#- z8ccV1XEx-&txLryq6Fl0Tcr-_ju}FVs_MY~fHJg8$G!y#KTLp2SK@d5%*j-_#F6U%%zi2leKI7wYk**X6?a4qaeiY{Xv_I7;`~O;i zT&86B9=o*WIB2oyIi+~5EywAQ`;%yIuK^rX@)o!N3m%=E4QCH+|Aasn5Sjzr&P=oQ z!{NVZmynzBI=sLmoG_;Fa#MVzpyjrL(Hwd?$=Hrvpma~^oML|wPN8~HIM&`D%(BAR z$->ju948Bgi%qD#uj?jUZK>pjet`-IsiIo#i)kXv5(W|WNjvxC_z011R{UWih%+ws z&!yN^Sk(!9ak0REcQIG zfVm0&aS?7}X*>ELaEd-&-b0r?7{WDUbtw_sQHU$Ui9$n5oO%d=woQ%koq`qzPJlRZ zXp73BIr^kBZxhDA!!m#cGesIv1K5J{$zewlqiAMMyW$nmPPLhHdMs6AS|6~^gABF%~?)fm6>1_>sDcINL>o(L^E;*2<_I^ z6(=a@{L)DiJA%G5fr|XFAB#gp`+Mv&G%v4qHV7r)?XNgiVPuB$>{2Xo4g`aGSDLs6 zpKA&W1jdvh=b1vC-N>UKqWDbf$JkwjbHtYb3XB?!WnxEscs~ETR=irt@eWh=xC@;? zNqeV$|Auf8aUG;PBIAV=ZNba;-Eq-HISM18+Wo5kfWU?Qd*dzz-4wnAtDVu#7Xw;c z(2>hOUpltgTQDV0hbw1^aLF;w!foz7@ycEQZlx{oF7q1cIj94Zh2~#cua7nlkG#p3PITibZih3cm z8q;9>bOmAO+tkZyPK$(bSAtpYi!(HFVb!ZZivn|wR=OIp=tKEouS!h32H3^X-QCvM z{8|8Wlup{_SCdCw59pf7>n=m^0ylwZfa1>ZAcb8^?Qj`IVhuq>7ip0Zzcd892y{)+ zEa8&(W@x7>FZY}ipCqKZ!e+im{ea=+JmE!aN$E1HMpNp}SmYM|FItcFI*yJ6R6BiF zj1%cK3x|C@qMTZz_O9pN3T^(?T!_Suw}HA=Ec(#4^~EDK$kugu8NZ!a$YSsK_^*QM zQ0n`0Pls?GEWD|^X!ylqBmQ&;VdvM-k`A=xVdcQPK%E~al-xlm#lP;(P<>ltr#3>% z+>;@O%e0u@{$3!91dc&DRY2W6Lzr2Mk5V0qJ|@HkYSxQR+uoNGLg$p0Kr;nZqhOYH z^8GoH&IXy+#jipvL`WK4o)s5L;h_%!R~g9Bw)JUmKMd+}X%9oM>WSqZ0fR>rydzG2 z6vC}m>Xf_J*2e%~66PZD$Hxg5?BIHa)FO`y^2Q9)=fbI`7MOp0f zv^|~zvalon#d`@WKb_+x4GCEddJc$UYGD}fh^Tfvo#lQffO1QV+Wox491X|66}Bjg z*H^y)DJheU$nh^h6mrkB(HTp=0)U`%X`Zu)heQ{x(?`n?~-z8{1N=JzO1XF=9m5Vi6{8A818H3`8_xz9CXvNy^ zLzvwxCEDY8(JFA4c(CpVQ1AdQ#s&&k$FCOdrsfY3rLNfbLui+<9WS*q9UA!&kh#Hl z!Vq}c#{gouyqqro352^u@=VQBP(qx9oV>{dWV#?HLGhRu=K2(%7&Z(420Ea-{xe{6 zKz6Bkr>ewqpMz0&tm)$a$U5sN%Zuaf<1PtF;ch_|SQmo(;;spBXXcLVD063acXqMG z3GM-c1OfziC%8lK0Kq-M{fCG5^YqiVFXz1H+&{XW+uxRIudb@TKw+mdDXGv-`m_L; zFEaUk^mzf8L+SkMd;tJ|Y4U@P3x;`5i8*Gjoag8ibxPBqyqtKDax)&G+~G|f)fg++s6AkGm@3A&BUni0FIq4=*LwEzl5Uo2Suu2tL8Cg{Ci0WlZHwP{ zJ^we3jnI;rM}*rd`FFvNPe|rH7j^Ki7FYkaU?_)Sb&TcDhCCdheDtvMpu#S)UMB8A z)O)HRug+6D;_FPwB{J6$iMbvaMaQuuJ1rQuJd(G*T0NY>6sRb zE=$m*K(b6nxI=(bkP;0%+`De3086``FjcBZUj zcM5O>)>>ZXUJ05;)te--#>x;>Yp^5MQ7~64SrqGzcg|J(|N6g+eHF~3k1K!pcH^oA zVNR{!`DesxAm$`%N(-;O$K=(4a`u|jcN+#x^J*`(>{D z$H$a($eecoH0ahJ3uvAqfs~5kH~<&XHiAAhd?46`C|izLbUdK@f$k=7{D*PX?+b>T zY|b2XxrKz88J;?m_Di2oXWiA+JJb?S3xI#gh-MwaW$w{|iOrM>ywtk<;XyKbTr9mF zluN)>j&Evi7UpyjrBI~@)(3FYaWCJcQvP*=5{In{!+zrp0bD(uUY^Nop7&sxg1Xu@n#AUR^w2zybGE6aH)9UV|l zVy--Svl6$pqmAP=f+9h#9a24BYPxD7fLVawD=TBJy%iXo#4$k}yKVWv)6A<4pwiWc zqvxjWiQ(?28J=sc({=!<_tP{FBvq-icPdG-J0srB0@UManDurh?5d(^nw+d1FUTFR zzcMKn-UZS;o+3A7$*yqft_953^Njo%yOp$1SCHU5P7>s$xkt(qI0i_sI3BVmcpker ztALy5?p2T)H&M(sd;Fmwm70v;W0!qEm>iJf|E!=B(1H)w$2Wo0YoCq;bi7*3+YD^Z z`;@j<3xI2kh)bRU;@fJ8LpNhD_V?BjhgmzRq+BP+873DkcnEC+p)`G6aY+Y3v#f@h z?>{F&!r(z%6TjO@IQQ#{94x%m1>l5`Q0tMI#kw86iHoMa6|UDJUAZPD+I4z?-KQt% z)ssp4`4jAhVv~NtPEeMYjNGq<CuB$Q#DRpJ?KIv%i~A3PcKI<|=SdH?)g3bm#{EgPTlWZu915n*x%$|> zhe0?8B;fR)TMh^Cm_d-Dm(*zZS&snBJqQ^zTZf$*2FQ?G;Z1q>EPv;|c|dJDDYPhm#1Ct}G)4;tNqO4nA>oqDs^7;->)T-oo8VCC(J! z=ITV0`BCG4WU#pT9ypc%U4C{?tf+CI5VB{|-u^TP(Sy8R!MsT8@P^hR`=K3A2cMzLE{sD}u~awBV)3(GriJ+rZfFBpdg=&jZUP)}RyVlw^mF z4M-dzh+La>tXll9B?)%I$rD1XazVkG5hW!{jOGi$Tv05mbdKy5lGiar)Lh##B38Hv zF!!onIn?~*;_{vL;56C-mz3}7w+Fm;Au#tLDpHMMe+PgYY|dkeOUs{Zi`jk6a~VK= z>gtUrE{8O0@@qlNPgnZ;X~_EbRoLSt$wN1~s>IjSnN;x46XrNg+VFiP+66);OS>L= zb%}s$wehjxwIxEeqFejv0$hrur+`{u+#}4Lc?w?5L}~Jpl;eTxAY6wY#`NqrK)Rif z=ChaWkG&d@b%$`*aAWzX9hbhli6p^x=?OkB!Y#x&K^rKRy1As8u1D!H#|m;kqB%p zo)QV*{f?}wqZbs;Qw5jDh)kGe`yK$Si?{5bK>I~@B@gn43nSZZGG6r|r&Z%PH0hC& z7BV@#g~<4vCeWq9?GUoF%A)`@4*EmYI9h}gOlF0(Sl}@bYN9K<`Tt4S1;=Y|d$fpl zY#LIg-~SVm2OEBGn6Vj2Yd={)O=1|oV2HZvQ-I7roa-hjNEV&!yf%5-VWb=Jy236h zXE`Krb3X&%{P7Vs5Wf-a{1HOp4KX^N1)(xYCOIG?&rY09Bz$dqu8~5uX&{al?T$}h ze@avy8a?cSbwf7S3nh^(&xz$<1ah{#xWY|Q{7DEkpjXDV|0Z1DWzc>y!=Vi~eYwDu zX6^;h0#$o`AlPM;;fDItkFNl^+Q}z$eEKSYnJWG<%~GyYUIQe{W@IZ1zE0TXmaZP6 z9lV6U>Hl@MoawESK-$VDHRkQ!2CPeV@{U#u?qA*k_qb?6jLB9-7s1a6HS2qLgera% zZ5~K3iizj^cR}jhw!@=^|5HAK1-zXgx?U04gzD&g#RnxN7|cCmto|WLorC508#fE~ zOoZ)G+#}NY;i89>7!Ds7h&v>$NAa~Fmy^e5#h61sDM&I6?~1XXf|%jDrD%`41mwz9 zv9fE8H9iB$vmo8b8GlTyD$2j>i}GCy7MyHvw)?Um$$xq+?)X2a#E8^$ajSm| z=~gJRAIXmn5KhO`O)(lI-<4#fCr-Uq_yNT2fn%l4m@b;`(8Mj|n+m%=D7BN=w*u;X z&mLCe%^v~X^~a2k??pLEkxaz}GYzk2!jZo}?iXz?=yvJVSn(rhvNUIC#H~F)I*u zlG6WY+1gf!xy@k3@I3)JSK_eTovJ7lD@$S?Sh0M;{(%PjKLM@@remo|H&_Ws^ETb% zxRo82q0?f&Bka;L=G(B9ABgo<0n0s+=u5=doXE?iw4+iSu8wzx)h_dN_lr1&{%gtG-b;eo^x~YRL;QFE|Dnk4C zPGQ%hvr4gI$&CSMdzlNxG9vu17IZXc1@NhqKt5MEH@G!)V*gG2aq843i7=OYFrgmW z6w;+aV$KU$qDiU^A#m5N>=#x zj<{n>Cou+_!$)(tf(CofR^T2FYS?}jHD;Dmg}Sv+%*x6KF9~#-%p36KsjUHA6lUjI z8=7@%Zd2mac)kf0Z~?AjtJLeh7VZ34Ls+wuSZKTL%4ZyOcocp9?E&0|d51C-O9-PG zH0{Kw9U#ofDfIZIcZ7s&`iN9S9uI)KWye+^*?CjuW zj>UI|&Uy2K8yeZwxKtQ@w-5Hpkn#Q@({ZUA$u8awJGe+y(eQ~K1`Ui=+b(1 zex}ivMe@}i|Js%R%@u4=d21o{tlM@kDIu}Mn+_?RvDF;(zkSPBEpj6y4U?OMk&kZKr&kEO7z}EohJzVy4Xk6kU*D4 z%9lx|^N=vtkX?S<+zhGe;9KfFg2wPJEOZ5nhK`t zwm40c8WVS{BOgW19%0Z(`V1y zwHVg}TB~bbLda%-o}RpP3pw zD#TscxBNV!Ajr(AFn4{`h{5(luYbhQ{v{dtum(A=fwXc%339U#Hw&WCj(A11YcV8^ zq~rmn{~$+VT&BI-G|_I~v{9Fbqi+u`ALb1`6?t)nKVMQNVPue}9_uhRY;nf%gk3c@OPpKb(Y_}XEInQz)ujn1`WsEtLpdx_ zZ{;L#7ZwXd+MaEP;6tqu1UU$DIM7fzinYp~7Zc7kj}o&T1r} zN8pnc^gQPQx`*&s8Gg-0J3)zghT{42O9b>Fc>MdX(Ah9h*I8O?EvG+M|2U~MWM;jf zB*h_(^Wm!k+|99jO80GBURWUM)aU`j{sx5V7nh&?@mK?D!X>9PL-+d|AA0c|8@G#a zMX`m8O)i3T|6oGU&5<_r6NQ@B+*!-%;p;-+m5zXSTte7oqx&XV&>DXSa?_$D)glwi z_`5KD!eDB@jIeWU#+mH=mzNZ}xVRJV#t3qjlGk#m8+%1T@=KKKTnR!&(v{|&BJxC_ z4K;&m!K;AWAUKtx=KTW9X5DDTtD;GOL(M?!dUc7d_dK{9xwZf`**?Uy>kEL;8V#~f z1US7B@*XkEjnJeg%kp?O6C);|WVhQ*{9}foe9QF$J$SpD!Q3lREk@Z!f}ewKbrS3| z@jiYV1fzj%#l+hQ!accl-s&HOT`wLI$6vtd0xW3dcKsKR4ENsIh{<~tJi>EV0XTM` z68Y5K0Iq^wl!~84J8e`CX!{@U18_q$GeQ|Suig)0PIF9Pzk#Isfda8BUC!pBg}MS_D9N}$eKqDi_rqS#~Cv={{ASG%h0OFqMPGWA-My= zwSPSZNm^358+hDdU31KMf^cr|-WEJoiH4!A{jube1oH@IZ0`FMq#H;#i%IM<+tXlP z#94Z=g!EutNw8~ze+#C$zGndPps5YX`$R7Xj0OeUNUuZWrRd#pac3 zOS#CO`BgyIv3pQPuk3Il#8*>bc2R|$kxCPYi;d3Fmn~jS3 zK7uMOHZDF9?SDIaV)>67U-zf`|BZyWMY$f7I#2xblLATiL}m%IdGsQ*lsjciO7mg}CJ8Aa!pAaIx~ z>{#imk_d|jZqPUa3GxSUTdULk4Mayl_7}5#O&n)b??f^F|G;v4%K-7QZ=fma;I#Nk zK_|&Rk2h=L-fv3+*=iOal;455&m%9wKa9uP8-gh=Zc*YJh0P(}eBnGd%l80ug!B~I zDnFEenJY#Pi>C!RUldPdg!qgQw|ghsnA(q!?t#AB)t={f|7(D_{qU0BDKO6hI84AF z9&^*rV3Z80d#s+rhzH^kh0TfP3hP^SrV;gGU?ixc-Aprs@Fm(TvDPevU50d8=}{Su zo3(s`rk9NNIMUW7D=gqrTI3dkq41+{lH%?wVh+M^BN--NHYbGBWkmORTe{ZVfM#GT zuUgx`+6`^IW#%if<)E_`R|qis`i}854dD;r3=U3?v6~EJTM1cE4sqr>0%8-$MS3<)I}W6yYi`p&`WY z{|!LdxoW>C4)z788;m)%n(j_!Sp>|LWm(k|q00(!d6|FWbCE7D$NfsQFA5EFb&XCnX!^JcRVSI zHm>x>+A9*wLpk}SWZpSdn6pdW_0^T2-BMD*Y~l7n7X*v10_-f9DPjeYWTCfm+TsOK znyF~MRSCjHseVsbov@RZB3n!q?W8NMZOk^bIPihPfa|S?`vIjmX$>$s52BqhF@8;m z%xJD#cpzAS3nN+4Eo(V4Fv{khqUUvkD)7QaLSCG*wjAB-XDsl8k!tR7g zb(g;T{%HR#KX@M~?7t;Z)ta-@SRhK5cDajhI$hs@Y8`mor<2UIVTqL`8!?|C&t_Kg&>im58v)YCnvo@M zLXbKlu5P1Fi)`wD@W-+5W(3{Tj6i-AW1h_mrt_B`B8=Y(Gncpni|ra@H<|&p#urGF zq&mmLLR;`f?#rmb#tx$4M6&UZJw=%lJWedD;Z`9Ocd$9O*^(fA4L|pBzrt=&Eo0nE zZB+p7JvfbQC%{RyV{+Be5o>M@l6%o0-Z8EaU2j9)an!rtJdV6g`L4CX8&teACd6#i zJFz@{M7c(6TR_||^LZ=?6gF!*xyiUoq>Gb0X<*p5T_Xa0{27tCbTW@P zX?tkbP*PK#Hy1#k(4I8D7vT;xV(s{B2PpEbRj0<(9SOS+rg^J>Fg+b{yI^-BN%onG zV!oY#NDzgYj<{lHh_WEin+J8xy4fzEE(DK&aFx&^K;LL%^(PUzW47xg+Y=Y>T9V2B zVYR~DA(TeMa``0%{f)LYQmwvu51@Siq1|;)KE`jkXGu#emMj^@UoTX3MUmr#~3Uib;Tn^vc4oO|K=cc5wVh1$M zq!P_g*q`eeV>CRuMx6xaY;`4&o|cZ;I|1DZaZezn2klBO>;`p>xh_HY_3Iu0($>2Z zH}6N-JVeqyDqa%pBy<4Fw!L2ptq;bYII|xzk7##7N_2xjrOG|je-w6!Ie&8iMVB;c zV9p7JitZQ^%@Aa{VWm4xlp7t54$X9r32`ReW#NNLYxI8x0m+2ZTwFMrunUC|Q9Lfv zZIib5oI`e>0^}?w_44)jsnF)cD80V%Q=oz{n#u|G&=9;&(R{G;3{Z-#%dO`QA?!M}54FIDqYedd4n4@bH$MW>O_4-b+SSZ`B$!#n zE|NDdhGL{33duIBH|`hV#$bynHya-aAxGUZuX04r=_b?6;==2#3IE@!gwfb?OvD+zx^W?__DM8Ftrvl|s=rtDiMW=%~T~>9j zza{+I@(e&1hF-{G`stqwz}}l_;*&E0oO4dm20`17CyP4xG_lxQ@MWT&y^ zIVCB?lr-!W4FLWl9r{^79=JTIz>AHs*ttNosa|Qv1N3<%2{~`5#zyA@xRLt?)dABz z@kF5-zzA)kMFgzD`DI+Guv;9Ty?XHV%LbCxqFNmFSK^oj^fi2nT`M5>KPi2%m|p;5 z)=8|0(1xBeLtwo(CEwj22q`^~Q)W7vtZ@-2g=4r?;tCOY)N@%FKV1xMR&tNY7-v8p za!J7ue`pZa;#FZ(3Q<7(?(YsynKC6VQaHDLQg0uKn*_N&lG*pg)0YBJfIi6rJ{D2$ zUGl1|33u9MWr@HE5T)F?uPOj1SCqFW3&_(Z2B5274M{V~ z$f6d*Mdi^UHPeZrT|_B1@}{W(hRa}kOum+&*~)#F>={110fh2v`HRJGgmC#~@W#ae z<)13h>EpY1LYyd?FPJ4;V)U&L7VB|A!}d#_E3{91Pk%TmoB~Lwigf>dfi9Z-UJS(w zw*zDjw^!57dhUM|NOMtzL0%P9F9F`ld+sdBu))Q7*f#>qpC$~E7r(3gkMIdnjyn^`uv|M?DPS zax-1VJ0fX`bj>mOQ4>iwYM#d+NCnO6cw1qQg7lUvHHy9d3FeHk*h+lg?uimjo)s#* zKNs^p3Ff+S0>pbwI=c&WUDzL`8mnzrdm5N5i4`NBC0tkE?U8;A8zvUje+YMa`L!f8 zqFxi|KH0`;htDg|f#e=7d*P~Rmlk_r7VDunPY`{Y(Q>1Lo)~Z>B=62T+&&}JtZGA? z_Q}5>orxUnauf8!^Cd!0dsiIxLWv+feKo4ik1zsauW~)~1WB6t*p=J1qQ?I=a;>OA9)7U}_bM^bAs5kiy=v@7dt~C3< zCa9h~QgFHxO>p&Fp!5eFY2((n%U6;!spXb`r$E>Xv`fdJ_FjP)wk(MQ@xXt9oOZge z;E73+`&R=c-az|)gF1H|^geOnQ4J-_miR`Lo3|#_4ETtriVq5iZwndr13oR#jG#d! zbiMjRkUT_Xoxk))(B}Twk+JZ{P-c^qr4W(#$GXBaQYhUf3cIzr_>H|9(X&-7&o+KS z6n9(M(^grI#HK%d>QqK?vsY0V{z<_mZWQ6lqyuZ(zkVXnP2uawq3HO`AEAWY9p`-x zk^3?iWE_Lu5K?bo{9DNB#KvEM)mu*wMx}N3v#&v&c2l(j!}+#Y`~QGkts3&Z6Gb}p z=6+L{{gw~#Ysdh z&KBy{p4=M`it=ZzQn#YRtT)rh6ahz8Fz_}}&J5SP-PJU6-Yw82Vx1Zv^UMsL`m?gICgxcD7c`G2o+{ek%TokO0zI+z68^rw zqciqY*fl|B*3})={HH7v&uje26vv}d151|JP044#*#h#w7^Hs47=$qKX{C;*MdaMI zuF!xn(^BOV`6_@-vkK8%DaFu*mxf?9u1$*#mm%n~S+j3P0q{xAcvf4Mu#-kHLDOto z+#oD>P_oxL*%DtY2k53r(lNQRMQdq!P_ubjZ*RQ00whJ?5@B@Qyds1vB~@$E`1MK! zsZPb3uNwQWToA^>V1H9wCWOMVafk(1A?(!Azm45jgLDzl3!(IMcwSQ%>THmAHZh@{ zw>pTYt`$eero+nr+{eqXdteO^mvU6>y%vNSVBcYsbB^I)FmIqaey?C&qd39K!@DF+ zD~|xK&q3;_;@fTlofJCNL-wyj z*g4}LD{<_Ib%ES9r6=7VUkNa~SO;a;{=Jag_jxW7nfg%tdcBfTR_%*ng8U&ev_Z1F ztbvRep3GVP+xQF>jHI^um-YF~U4;%d--b}soR*6vHzHhSgb}go#?USYz8Z120TVtF z<{BjfQ$G8q5sgJxMwHl`Zc4OL%f5lA2p}KyBI6o`UGP>ZH({c%Ifxrn{sLnQNM~Yg zM~Ji}s*E`oO!Y(#%+5d%)77x{}&Y zh%3}7PhyOm(*-$o+;sRM;t4wxOj-?T$Me#TU@)$aeM0L_B^hd=j)6A~AXiM=rj8`Q zM3KEjWLv8%jc95>wgMT53A;FH1pD#G?u2uj$Pq_NydgwA1K*Sv?LpXmhMSyo_AGJr z^a(ZIV=oZ1MLWKj_YWloJ8MnOCkP=0bLc@)dHq!Bm@4tozBgd*Gkr46*hGM1C0SlN zKC}yTQShervM3iDy>HDvZwql(LKYX}_kr}phil{tMCLiua}w46wF%6*_Uf_0cu@fL z=I$`&Y%YJIl*OK9r#1jmPtzD4mf9hmJ&e$3ZfzjaG+;iNxsy1~G}-OzrKihPo`MX9PKBt!}kW$DU;hFtw2T%bgT<&KPCH;Ub*~r#)(bG<2$Cck*RV z4oe#OiNY>qa-=zIsL(76oiS0gE6OY)%>Wp<%QQe|hquSraC(UYOU>n&C{SG zsh}e=zxTyjCquhbylEG^i*(m!*EC^RoGrlddCZ~`N1OufHs{68Cn_g$A&M@AncM;5F2Ow?$2Q4p*t1!b1gALfb)>aZ=508A5GS#KpV^tgt?rE^y0QN z3)zBD_RnV(5|f;GS)@7GH`v!fISx7-#GRWPIxAh1^J0G~5t~r3W^6AbBoAflDM=S| zTq9l|r9u3#ju7{!Npjw-CmUn?Mzki}MRB5hUI@HsqFM`|N7&s3F$xx)2+%dX+#}po zYr;_Ti}NAPVeb%m$lwB}B2~?Q|BZ0HLGZpj*G15-E0c)$zjFhTJ`2TvN=Qz!4|_Rk ze&8iQEj%l&a-yyXIGqMzPzd)$06Se@lUmC43&!td|!nBKd_i_m8 z+{Jd`_$vszvU~Bo~B5TJ9Av&A0~8J)n;%IPK1UYG9}{c#nJ5 zwZwD9*y2gt9iIux{HaW;#1hv*Q?#Tcajpnwk62UpV-GZt-l;=Ovhn2gB?0F4DZJ|C z1UP47$#nyS`7nta9Gu?X2w;|R?HrJpJ1d<@WS zA1bH>o@l7a#3iBTt>u;sy_p{-CS_pP7_%zu0;7+bT)SNUNP$ozPsjFa1myztvJk!~ z+I@rD4Q;^RX~aO6r7!k*lo+{9*{>$)$$P@cmo|;XA9EP-YAp6RVMZ|C+BQ+pJ(u+m z8Ouuoa#wBV(b{H75xy7b;!t@W^m(Fug~W)7cl475;PEOhcoumIKv`5`DFtoWQtLp@ z!2zJk>5IMjnfm7!E2XZ2)CDdQPPt_)cjME9T^=c#v?Q75R(q%Zac0Z#~}9 ztT1I5>WEGSbIfY1B=vEOAh)pYRM~*n;;j62b8{u`QrLZDNJiT&vEXYUZrr|#BpC|} z@O)5}Hhr_*KCUG=b4cFec6%L~`e1-IZfv|M%0)!y%%VAU6I7ub{@;5;# zJ_?2}E9i{)fLApBm@Bk}$@4LoXlVaod z2)mwm<})Q;QYilJ}i}v>_f;*lc0;rbi@z7DFznbJSuV z@ITdOIi^VKeB>{u(Lg;4`%7si^v4MT+#L}9U>d3!Z_Q6iECeSQvz;%%v9g*ov2Onq z(ACiF$24)!XCPD_Ie5INFua`HAIE%7(Cwc5wCP3azYEJPHh{m24@DEbw-Rf8Ns!7Q zw$bH$RitC$?qDDe5KSh?dE$J9-LlPklZ=EP`U=QH5;-XYh|6fnlAiw>9A@G)q849> zAPq^vJO7Wc>n3ll=&#ZfR{RFgd@^v#w*bzmg)@#`RQdEfAd1}2Zu9#_be1FDnEwI5 zLkJb&_~Va|o#n>2+V z^D~gkDP6IlIBKR*_1aK<78wM^vC}LClpcA~#6x}$2t#BY9Dkh+!kM9YsR!Lg&0Y|> zt|m^C5I*RvAkJTZP6#;I#GY}-xd^-W@(dIL0j9{I_@B^R+T^lmD1J4!Q^06!LQEA= zMi>i{+v(4O%{lCk;vtc)Alh@ohsB)p0Aw>m$>I^)W&OmR^-pG4rCYnyDGg*my#uA$ zd5Pmf`8?JfRYBCGjec>j!cLq;p{J&Oj*Io^d^r|psMF>ziRdC&>&x|mNMwM?d!q%) z-*R}YX6hDX4)x)Kg!d^mq(t=rv3`p9i9akGcVDkjc4~bL*d*Fr2#;d zHu*JBy{iqyWg^WmM2YB9eUSg}#*ZTd{`H0U--D6sZgrzdRDV-2nUD6iw$@@$m~-OR zfRTk8p8%o*Op;OfX(Dn-dUY95i+?Q)y^vHRjcuC8=>UT=`nM*UKx2Qv&sqAv0IsY?QC zpmxTi3OYfZAvBEpmnxqigU1)J=FF!B=5?kn*I;xlyL5?*3_K6v*96i&Si;90%Q)P` z9I(`~g!8mF*wP}4m-t>-9%q^e+GDTfK-^wz-PyVgTONSYws*&<6-sQo&W;C#Ftei} zh9cRx6`?Z!q=kH{Xs2Zd3m0a96GYpf6m>2W;EpKGy#D4PIb}b1Rj0~AGpQr0t3lDs z9F#}I#UgTNPg3NXr7~v>${JSFy z2FWH2;lXFZ+*R=Aq74fDCq@;pw`o#VFK83!Qcvz35*z!3cz-cw?bvHHR2JW}yY8=Y z_mu=;3ydirqKP2Q5h;8x@5lvrhaAyL4I zHAFg54%HnD93Gik8%zzyGCWG*)MGG(t%-1b*y6D(cufGz!i0yJa<1P)xskccpB8hA zb~9uhV-g=Oki>f_>z)cT9$K4Y<8=tSBAj7*SY-aRZuy$=KF~YBTU~Lsz#L8PDQfYz z^*|_~?rmf5^~*PvZtajz0FT=?03=6=Sq|8cu#1Sqn?5Sq>9O`}=H6jrklaLh`xqw( zbJH?YGhVClfuQ_hrBjvpy73{Ll$JTqqisekvgUp}j!@W%=rJ(0-YxlW*5z1cbN)7O z`vt$qh2_Z`)6>+V?&eCEPq)HQkoN6~m ztfxf!d!!Bc(2FIv1aUc#cq5I!u>sVk$c|*jRwYJ1Lh_;br~&BFx@w%V4KbXrd>v_6 zUAb+^h?B64^=8`+go5H!A$k?gOy?>#ZG0~i=H6pNGjVLb9RTZtrDqTY7D@Zj`xIX9 znS%%kPZZ!`jNE>a9ieiqIYX>2vc5mTgJTvwJS*5aqSnEtvll(QcLh?+L7MpBod`P@ zH2M*TpC`cGslUpBuy1E*=ZcQsN4r4f{@;W9n$#uM+ZE7nOeDcV{kXvr^!wfT%axOQ zA$kbb*Vsh3o1VG0Zsh%i&^#Sbm=5IpvD)rnE@0wvtg}Y}xEyAUeRfX(H@L4@E|8!=ZFmlB@1l8<9Au48*d{B@(tUHE##6Z2@Fbn!=l)Rf4%&qI{C3>oZ%y{=XjB z67LCjpZ9f6OZ-QWdmWbzv0NKuZX8~X8Hfi3z*nhJ|4(6Om2Rih*-vjT7;inJ!p=5! z?@3@zb27)plSI11F}Hsx3Z^7Yd1i;zJE7g!cobBUj|(BbJdw1=kD_RzF5N1w*5$uY z5MeIAK$Q8*YCIwayP-?hXK(e0fYP}K*?_VhT-?)$g%FfG(Ov+UIXE@u=_6R5z}Ow^!;CNCyPfi(MJyMFWP-rUK1;`Cvf=SB))%nRi%~qp}_gO$DK-x ztJRj6dm5N?YfpC`wK!god$=Ck?&dMw)bXbS*IAJq^y&a}$Nr#hT3JiqaDbCTp$*j! zj;}4T?14be6&Y(64t8sCi7MW_7`NT-Y5rxcKnXdr%>G?qTJeA0%|&0Y->F--hXBwbIB^9;n2 zhe4TH+%BfttsX0`dtk>yJqLMJ7fa|D^qy*xhNkBc+O1x6hO z>^{U9n<9@n8lZIQv2mVg*AYi4(SKY?ramicV}GPzVrdmZ{CtW?r`j^a?mJqK2XLwb zGTPqt1Zet+9=2#xxIV1ekVx!9@~h-5`hbk{-~V6 zcP@ve8EG)i|L2lC9%tCme9{2;66gB=IFlGI5>icx)t?dK5^!4~%bR##kQ0|qpIk-K zHnW{oKIPmYor{%)z!*fsamLw%J)4dhA3yvB%2UbsSp8f_)6QkY`Tk={6;-r1{tDp+ zk)N?70zTmaFw((1tWu53MY(W#i$=%R^DYE(J4>Pxm;MbpGr7_%KPW3+4B};G5Q~{+ z)L?4z-}=KP;O??aCUM+lkZudzIo9Hy%K@AjO9n2%I-~CjAQ~1&k*6t`MmYU;^Q$1t zq2%*YYHCc7SA)V2wncGh7<0GUPmRpJ^^E;lbZv1f@ph4_;+5LpxsxDLog z?qm`E$Muk|J2#G<@t$ZiP|~TYF4KM#=4wgy#VeLK061B_d#jm#H9<~?DVR6mMLQjy z^W=EKmESTqmUOrhXE%3<0H;$)YUh>sNKh^h+l{#W=8}%zl%+hszXi-w6K4^5H>LnX z0-YAyvh?og2@MFH!em;o{jJ0@i+CPnpuHA92!X?TIO2rc9ByOG?0S1ikBinGnPhk0 z0hH&OBz(M30N--Li1q$Kn8r-~WxhMh$JA7M>d`ES|I>2Tl?mi?W8zQ|q*5E|j%O8i z`E>n954hlNkle+Z@p5wyr0cF_h%0cI{j|XPxfoqqS!*XR2z%XIV#`yoyk=D60D-yn zvSaGS?kn+<+N3siaoq!eZVH^oHb?Ix&~!E(ZMx%3Q63VAm6IK7()4*)xEsE?H+hn5 z;|8i=^3s7KIohoa49y&>!Lj7vM~Ua!qb@hZqH&@SYVVnmhcHV7rtemU&5#fm5=P_J zRDuW9c=jznH;_s)bBR5kC623u zm*|1GQFNZq^rVljAfFXRw$jmyUq1&y9?9U22}h(`CB1%3gQwf}4~4tiFgmac(WTFb ze*xx$7t>X;8R8fC7TedD9CQk@stV31WwPZagu1uwUIg^O6)!Y*ar{{pxOoDD|DrJ}tij2#zOzYpaO(Su== zt|8$n`}GgNseC&}p*s|Ip&MGUdT24``jCHI=QqG4`Q~D=&wyQH zwy%A0+vm`vi3<~63w5}Yu5*#XE~4J3lQ3GUP2n%$-iQ{8PVy)a)w*z<5Ixg_tJgtZxasnzD(K75vQK0l8ds!uHs^ zL8GM0n{%SghPo+UOI#w546zND)$FkEAzgCC!B07-mmk2~oXo!%r(P$3a>$rdWlyXiLg((uv4Mj14#;etHcvax z4d{ANK2EnS(Jm_ntb zH}AaV3r;-pMkxShXg}UGA7K}c1vm5WEP*bGTu#OfqMbzt4_SO5vc7Lkol}~ro6Qdj z!{FaQ^oVc@lco6bq-Yq(Eg96C3qZLJ2+io6b1&$>xg;4oHhvIA`FO%19{3euXRYhd z%0iHNdf?JAy%xmL>3hNCoxG;Z{Tm2#K(a7e|2rZ4t*va}maJYh`bKBmEV{mv(xosjVXiE~R$iW71lol}OD^qJbuB+#u-R7k_yOCl z6lz|uCt}xl@1g)=m9{0Z)nZWa0z0?3RAJ2rGK7l-mw?I*%}c=axt1kCT;OhWIHx5k z?&$)}K-q>axfEnQJA5+4ga#l*%JvwsborngDadxqK{^|zb9L=GmM=+g*knV}DZqW% z>V&m;Nsw8>5Ua#GE0k}Tuu$>8Q2_PQ#%u2t37au3JP=%0yI%>E+E%!lY>Pug5x=9e zH=0)_>^9&qK)0m_yd%tIU>~8^ed7J!0nzGo`QG^Tst`I{_r)g)lCT%)%U6SPNjc9C z#oVhyyYXzxZxQfjg1unKssovjFO(M=2Chshh^G%&(*!~ZHcTSSxOS{=P|00%EfC7k z+!q@uNOkZm6;l*;(I?9wh%N59f^rwu?h4aLA?m`s`W!Dc{+5*@e?K$4q`{M9yu5=3 z;Rql%{3NVsZWQT;AE->FJ3cAEc{FL-?2p%kxE5i&EJ!uA5LI;LwwuU2H6*-n|$llSJ2hAc`zh=2$_lFHbbHh_APHiPMKkp{%+7Eyy`D z6y#*)-E}~m@Q~E@w_3M+ge-(DKBiUc15tC<@k;b&0-;`4TrR>{avw)APuc*$43+0D zcDSryFaY^7M`qI;XKc)$W{G_Y#pOcWqPkYY;qXA5x=D$J zG=P;mUK8Tp*3~f>%WMh>lPcY)$Q>lYX(op(+{Mi zx2-=wAuXQRzWj|JYEF&a@tYlhG$GLghhjBRF5zUhXS8;^klYF7R-5w75ahL`*b^mL z-!VJ#hx$LP>)45~tAY@N=haw2u`>`#41^f5&d!8$ch`BgDjH_zjs1qZK<0%?Ze7#G zbvneUp(XmfNVvytuo|=O=I^EThD1|T?gS}Uy8CUy z+>bgrE%wA7y8}2ARKv1^*7)xpB^vK9ae&%)PXKd{;jL5kg+l71j>&%D574fV9z)in zRkb)>pxN8XETHS7=LEqcY4^|aN5b{?;1YqyqlCE3cv+Y8q-tEhHyATjWomq+AhnfT zGS=A#!a2%$RWqg``0EqsVzF`Ri{nJ|MU4iES4GrY+fDZ1zQ2eMKQ3x2G>@2}6(QQ6 zAg1n+&fwC`KrS`Xb-HbQA%qgLaXYGopc$+~T6c`E0=P!H%SdcFP#ArILdnI!tgR+# z(jIiY|0Fhq7DJB9P<}iAiR~^31t2u3T=tgx}e=ZlKvbs2`l$Z-6bVt4@h383&_#f z>0mNQkBg&)=JFyYNe)%jNC}5u7~Ye7EGNW+-X>1)K27oaOy&5mt9P>A%YStY zOpA{db`7|ZMic0(2A~T~-aMOH62{PdC^|)(3kWAVu;f^4T1g(?gfUU1rU^jH3nD?FW^w(*pr+i8T%`U*k?8T^StRfLPfj!j0Kx z5Gzk0>FowO#5Ka4FBh)3?&d^x2oPl-pG<4*;1>KrNu;u6u0!)di6J)F1v z8DQ?@_!C+)Q#cuC-FZ=zGuDL(3;kC@a>i0)GLpYKaCHWQ416 z==A_z*6Ooc64pL1+&t>(^&46j-T>s`jAzy4?29tKBQ(z}pzU0_0f;4dpkMhW!me=( zmtFgc%r#{5K-0f=Gl;X1zLN&b#9RD*4~7O`DeMo>*3+6g&#nGf2HhJg?5ta*@Ef1y zKe?ZarSEV=rid7m!;*scD$IIaZHi|Vtk3LZ=ec>we}KA3T<2i4Rf|Uixfty{JwiJ? zB?J!Z^bw2S3E_t1e1dt5XgJW1e%-8hJ0jd487qq551GpyuCUWgx(Vot2+1>5I{D6g z4?v#EeF3c7i5Ws2t+z?7z3jaJ&K|WBUReMRevN z3U33^)rf#>e^eYTn$&dA=g=-nBa7yXH6MU5r_hG$kap6wf}9+ySaWae_F##FH4OWU zTKrd#%ZtawuJ~LuC2YqikX2-rhar^(va?uA1i2$rV5a^|6#Q%Fy{|FKhtjp5vlgGM(`DJ#ng`NQOyxXRE zsK?vvT?AA0DLkxp*^?y-Q}cix71sgfdBNo80&VgXRL*Y*pXt4bWVR4SjJHmo(T0<2 zQ7}4bkfB5SGi%OVpF; z)jUNdFgGr0p7H~`-m4`Gj-DuzU`)lC?-If8K3F)(QRN*%{4Hw@3nmW{yaq%wRJe2M zif2V-#w3UHqhAMbk8Ed14Sxd?&dSH*8*e&{KZ0m_i?FleajY6f7&i+cW(BLH+Z4{j zzP%;(cpK8yvZr-)tKt}exn~ct?TAYSI7#e6cqS+Q`!0yPg|)QOUUl?)B_3?!%n(!m z1CmFUZtk@m{F^W*kEa6iyH9{wftd+A7dGR9V1=1_?EB>lBvmZ0*EWC=6Go327Poys z4CkZgG*fA|`w+}^Q;%d}Vig(j5n$p)U(EM0VRwflK_q2Oq)P~P`M5tu)xE0$;NXF! z>=^-RgdoECUPNQ1)U{ViX6a8Hi7A_VHW6wp*7t`Mc3Baja^U9w?}eG4NLn~1B(K}c zed@2I^02kSZUYQ$qyw_aX9a;Pl7{Rd#Er|86eo!^UsRU)+y4Di6o z|3QlxtOJ{VOV~wjVqsOs2yJMsy!|~9T<4Y{ zHa&3e9YIbLaW{;6_ZM+`2f0W^AO8p9(4s?3z_GIkrj`M86BYE#$P=P1qMSI~ONtjC z^1tk-q|kMfAZO0TYV5FhtN}FZd7F6RCt|ot90F?i=UC={AY_V)Gs=fY z{tN}TdSTCJ3cKsHN^ey?deTf|>V?Aoirq?A>^n0MwL#3yoAV-^ev9TJe0RS7Oy??Tj2J&|eDux@P3h$}28+Rxhz}cL>TLpDB^Ftkp-_NXh^|H) z{u>ZYF=&Ry_}@CIZmuNPQP^F8nL-B}loO(0I6`E%Oeeo%;etuVpJwCni-9>Cyv|7U z^{fy#subPnFkJn8u{fantQ2CZF=B}ltJ1Zk%P@c*cALt(%2qLd#MYMyCneqSUr-?a8vdrg9xg!D3MY$3}1M203_;cgOGZ(Mms+Gi~Q%E|V0 zD4IpUzZUHT^S<#=VQxxk>LIdar-RqI5x_3R*wMq{VUZ*wCAnxH4dG1Wjd3VW72x_) zcLpsYctI|m7IO7to|X|<7_FIb^D!W9(R6CT^fKNSrnb@b(XL}5T#hMKj-?{q7(9Qf z4TLhAF4U%fB;fJ@U;oub!9xs=?!0VZ)( z>S6Z^G!q#zgR;lK!sWh3R8Cx20Z6F*+bv3_BXA8nUZg96NFb?4&oi+k#@2~z|1{Qi z*%H+3#{Hp8nx>i-W zQ-HaZrp=|cbp%~JvfXTnkpdinNx9v$Q$QC1hCZkWZcVh&V!7=g%dLUm(9!^$a|dur zA;H|^3OfzNz9?qRyAuGTa|(9G3X)t_Lz^g)&&DChctK%*hR+`cOzaFmH8H^(iUUNL zsWKCgeb2(X0GWNYYBIFCcvq0jzU-MQOy>#+hZ>g{$FoRNGtZ12E;j=oA>_Majz1q3CW$N(k4N!*c4e!RzRzYXLGVxOI)0tB|fN zErv~XtSrPummx-594sJjJpH5#*Mle$3wA;@T>LE3EmZ4bfw-g%+6nXEdna7nVp56G zJjnKFl}>0E4*T_VCv%!0SUA9*b;qvqH|ew2t;K5H{+neoR_Y<_>?d)bcwH}~`9i6E zdC`KmLr`~^yzjS z;lyAK!WCuul0VWB1LYI6G6vHVeh)V=&Go6Hy)4X}*7kB*JTVAB^;@t+dRgH-K{Hg^ za6iGMG1KINqO-&)E@`~nB-BNa8Y)ZXzl6BdoDdP*)6YAmly9>ODa&0vGPmseJq}5Vwf-E7cfwc=^gAzgk=^D6g^dU7B*7^@#Fe z4dE7UrUg(}7T1orzws}pP2?hbA6cTK+*u}!9pCn-)3UQVU1^YsY zw~z8CgS=Kb&(Va<73oK=AkuvpMQo0aqA3iT#F$ky;&xKdtNz4)t}>S09PQ#5fZQPl zd$3uIsX`nDZ;_Pt5&`O`%%!7_Bb-Mu3W3ttI7bi*VlUpe8voLS@D$rGLAh}lhG{oa zJs!*zZ0hJ`Z=jb=zc`_!+*O&9jKsD+5r~Y3I7m!Vn6b@t%XDynD3`js17Ag5@v#th z4V(a|18P;Bd=es=IVH!JF4wlFL8L6Dc(A)D z3e(3VFzIx{RJpS=9#$|jR96MD;GY5VK}nC{u*uXJeWYOLhSUPJJxaHWFf&uW|LD$f z;h6V}uu2pth?XkPa^6l6-Rdnt{Qmh|{M}@eB zD=o=)G}S%f+!9S^273A;=YizTAxX40U#ALl;`l&geiH4ZywTI#a+N?grDUtOh;$>g zrfZ6!^8w7+3fJE{Wc^8y^U@SmlY^h9gt_3{)kx*$_zOx(xJ&08ccuWEy@HMO;Dv-; z6f_-pHc%(r&jjX%<<2JQzwGT_YHqSlSQxtTwFOi1RT)}dC(K_L>%(fzW)-JeB&I!g(Ozzq<$BYsT`_>p{)MYF85%&{y6FU{0grEW?wS_oiP!c-??g zw+OP#fr8@Io24Pu{C79=hr1?Q@Yqfy9B0RlzQgyTTtF^0q=$UTEsl;tH!`kR=2l1s zb_=JO+X$K!x@%{X)Dy1@%RP{JMKTpM!EOh17Mz!l6QvHzboRc&v;wN7)wt^ph+KVN zkZV{>`Uj{>gyO5D497GOE!|}BKm%#j+oS#Qr4Ta>FQ6z#O}MipK6w!L&%9K?S@h3B z>GpC{B^OwCiKaX##&pC?cbU+t7f;^};mW2-N-FUm3!^5?3-Q0k|4HB`vzZO=;s0E8 z?va?(_7vh_#__!~*1WgGK&mQ^ye=fiKqoHVZO~I#hvL7DFPITfRErJn;|o`{saLAe z+X?U(N&A%kd^O!GoXR108Hkfakn#P+C`v0lP4CUwFx0F+Ecfr2a2R*O-k}I5xEtc5v(y-;|^i&o!u?*u_#X$ zRBrMR5*}UHJ@+uM%ff1bbL~e8J!1Um;cYYA?iS!uJ%yaIWQ%+#Yv(mH+E+o@qXh^oPc)5evkW438Uj25u}2eRdoH6aGo3J zP})_TCn)!ier_(}`X~G`vm>u}h~Q&A#<|XugmbeX-b|~<8^UrwVFw^N@()h|F$MJW z#V$`1bd@A3huv#D1LA7nJgG0X67Ad(?Xnzo3UF7zm^BU)nG1}ESos_~S&*9`yMRv; zwN0J{rlj(_w7tUqN}l%BwD125$Qi*hGWu8m3~bjzdinDZd2X4;D=m=@=?0WQpVGT(Q4gY@$ef2b5~>k9>={|=t+`hB-)(7n^-?j4=nIjiGl7>E%p*^ z&h$>>5j7|{&1%>mtB7#>HgRt3ZIB#lV#9a&*VREiCfzi}mcraNMIeX(RVaz_AUIx> z8lPeGpKg_6{`ZJLg+{Cut0)KuQ9Ot-3O5!}tUzTxaYh5}=;1zD7eBRF>3!n5qVg5V z`t`C9x0$rAtu6fE2PIlEKJ1SjJ_Iobaq>Mr+C;nAIPP#66#ChtvM^#p;{a*mPRgx<<3C36a63kii6cBx24FQhCF54ccpA_aCxbc@i zjB#IqP<*vyd@I6fp!mRCwEWiqv{`RQY_Fg>#I_lm9?psX2jr$>%c99fvV%>8=7MlP z)fqp213-=2InYh}mN3~>d$6o}Uqoq@aq+QedTFvGjOpJ&<*iAT;|-#-ftdMwKsc5r z?OH4?Qpss9-Bd)q4K0g1L=ah{uMubd;P5!!U0C=>!e%^n?Xi+bXT@-=PUHMniG2mS z9-O+l!02y$#O5sa{K@I{p|~{n{|Gy4IWgB;H@_3)&w2fWb?eTV#?~{z9e=tb8;Db8 z2FycS4x4A21)2hDqp*X*b@q%K86VAB=t&iv+x;looujXp{dw9hqVYPLli>Nict>Hk zcY4YSpNg}WLrsa3t`9R#G z*|S;2eivctGpHW5#Ggf}Kh&BBW7GK|)c6!@dxbNfyD*NMz5uj&*P|T+Nk1)ZTw28wzb5RKN3)ta0h6tT{5J|6v6aGRFKP>MtjIiw zXm06RO#Us9v#btbxhD^3%Lf?M7&&~z_*i>k2y&Bx++P%?wz|J3IAm>T>7-~N@q9i+pxMd}xP{eMm zf1Xs>`5+q7y+ORRa!Hb7^FqMOj{qB6#eCW_l8?H1C` zaZ`aVIdafS>?GQ~i1Td|8>cA^q>n)Z%P$n7DNU!8RmVV>|5|6%En zohZ@`?CD~aaUiZX!p_)zJR~V%pfC`Nt_@MIA$kYMA~c6B^m|~Jl&67OqFJ<)Kn@u5 zuLJ4Yab#u9)Hf%r`-^YbJe($)rqiJ(Yv4R5$jNAGeLcrOmVupQ654+v$Tg(&T%U1O z)V)3!>}Ee5`zcJ}(fw?Xjt%^WB0ZGDQgfcZ5qRF1)UD0X;ucCWx^&%jW5RjXoytb8 zgA0V+Hz^ou0ad2hTI?&V-YwY{vSm&?(b{Iwq7LOOdALLn-3 znhk_HOM9nBLuQ&V_wJ-#%W3U8L8MI6F_XQQDLAy^nAmK~l5h{os@Wg@eL~&kxyoW* ze^h`AEn%s}IyBJCTZ6l;>!zO?jaLgMNAf^V>swK7TA7`*lrXuiyDea@Aj{A{L~6th z;GtvI?I7G$q!1QMigp5scG&4i4*sz~X8Pu7vGn!?-9OQ^kh7m%b^xJano$edM`5Ri z-D5wFU7^i2bf5K5L`&R~idqnAn!v z9WYN|*b?Z>eR2arIXd;Ty9BwsdM0QnuGymiST*y}K6?Vd^T}$mw?#N7+)JU&E1Abl zdjaMSW)Vhz-0%l5=ibVSpl`SQV}ZDHPZxk!3v%vBX(JZc8z9diavX(@)IK1&9fo@F zPffRP5|&$@3)py1bgn9w!4g~jdtV?Jh|`kTx!>k$MoxwH z+=MW78&UqGySyN^`4C|4yliuM20(zPgmT?U557aNKTc1_(@9$5t;WZlI5kHlZW^(u zPn#~c7oHB`(%@FU8e5BYR!kzy-_sirrHS7Ind^Xp>AaoxO{0Z5`F=fnjJe8zAn>P4hnVOR1rg?zGYIEpvr^;wCf%zH9^x3fFF!(I7l3(P?vZ9X6vQk- zMWi#95ba^bJyY!8Amvl$)!$2sHydD3faOM>97MTmJdw78m zJ*Hu>$`L@hSJ!F`uj2*yGaewW=uzC$j|9qnjgH3U4in*+qsxb_Rcxu_@IL|MWsTcelj}k^mtDcMHC_`|p10seYU5`~K;x z&eQdlobprWoC2h25%ts2z80mS*~XfA$f*!6G#8ldNpbl5(~2~j*zZQhRGu&mrvEA@%aRz_IgrfVEM=HcYRCd{@Uxjk2`ju)6X2#G~HnntY}U zAk0J#xY)SdRtH$3SrpzBK)yV$7^_`K*zMDfLh+>{&5ObQmTt_D#|~b8A}MLbR*bjCPvF($Rv<6Zs!yuAkw05a);G4&HrOn>G|i zf%t^C9Yi=44xQ|bs&VWMKsh6hm$AW(^)%RUBtLeS3zJB4DDF})Hw1SGIM#SXh$|$& zvFKmaQ=sp4B~67zZz7U2mU>KudQZ3+h=S20%5ks=Hw4~t;+#6!+}IXR-dcRE-mK<- zQIM-DjWAAS2V}RzoG#Iq4*x{c*7C8M*D8P`M?_d1kqDhWL)Fp+$0_1G%@mCRMU>CB$ zQWfQ_3x(NOQmG>=(ivTXx{LFtvapP`pCFoB2X}tem{ON^7K}H*cLh>tr~VV$!&%C^MNeVA`E+T8A3ZB2a%?<2hnc zkL=~@X_5XY&w0`CJW%f0df|sV5GFW*<{d97x5tH|&8s1afn(_x04S`47O%cYnCi)C zb4#r85>)0~Z+~ngvev%I!0M>qAr*Y3Rbtmyia1GSj+^q=KwK!yDG^6WOKzmVTw^3P zIt;)0I#6wG&R%IMvD+Jw*DPk`hqE9-;oYi|deAKY+}gS|+Q@e+W&(V7_k&^0oq8CO_twkO!;QFJP-Z}*>tV+E5B_LA@Qx>;GnP`;V`ix)k59#uPQ4IT=eQAAT2sr9zVrN1hG8Ap&*QA(=VoFCxv_CY}I^PSHgtX=-QZ+8L7t z=J~@OYLEQ3h=yQR215+Pvjv(nT3Xrj; z^0nXiony$JJ=uXDEdU-2G0}bceet=}hIugH2LKlV4??Jx^~S|N0=Wni$3^@MdSv&>?XR_Kodrfh*kkkP?9(tKN+HV(zU_7zZU6DX={t|zZKuhzf(sXBfze}$~DR(gU&I;hcqRo6KQ*rXK7q`yFA5I?y1g1>dZpAskTy?fY z$TR!m9YM9-2ao!t%_APu=K^-M&@996ZX7f>h;v3{2Gh2m2Y`ObD+DJg2>*~wc0}{M z5N;3Y%AEU$;yc;yj)*Dq0n~EjJ|gc#mz&>daJcD>;UbDS7_o4VLKz+xoVz1?T0M)h z;sOOkLO>m{x1ikN)4NPET-|v=Kv*kT`Ed&u-{^fK*#_Pq$c;K-Z2UQ()F0TwM*kuZ zx#9y-$eC#|04kUjo5qQ(F-CH%;{>>rHN(OM7Y8ys<>Z4JUa$m6(foQh6yuI1!OS`7 zGj%TwNhx@*u_Iy`h}`ff50vBHVE|M}FS7h?+2T)5hopW@9pG(Ww&9OwK&(u{%NLg8 zlRQh{buE4tZH`NHrS}sxdgfgoSS^e4d@Liv-{9V@678bR=+uRofU0p$9pvsRO9Y+} z=+aBWt~Z)jr~@UeO5;iaPK)l^$YWoePNuF5Xig%#lz~xvCMeI5%5rRogKBAQ(nuTf-$sLApz8x}gK1|311%jAC&uh0V{H0!l{Z6!wj= z1(d9pf$kd*ln3O5Sbahfp;0$z6mt_n=3&yKta)%hwH}qXR+G)gOM;vhu1+qW2<7%d z@o)lfDnhH7*(`p!mcw$Gd+OSRb9LxtY3n%CI$&;-^zKbHwh=;;v0iq^KI=l5bv*ru z$!crd^5^1}^GO1WL@<71IRNX81N_81b?Yr_~|b`|0%< z$Q{%H58R9xq^{%Os+$+FTL*i(qf4|iXvP+krSK9g3tXc$XmOhCVlL+qN%gG_uOEv1p1WClj&d z4iK&-FZyup0n;|w3CwBswQ8S_ZQ#y8xiOe(c+p7!r9wS1?pD~R%Osvimex9E;-8g& z$O^3k%n35V7QY%>?#e&zA}wrDJ`!15K4rp#y%-HT?k~VDiM8c;-$sxNjBW+e=$7s{ zZ8tEph3i7OTo}D4i027}!kTm8-g^Qa4by%(Bj~=zR#f`06>|iuAW!`?SaG`}%Jg1~C_JC6a7-+ajha>{$RqIgXEM*UULe zc;+xlepBOh0q_o4TsihCLF8J?VGRz$<9T6p6LgljC7YoUD)%XN2pXi93nJS_Ch7+j zcJda1oYMqA@@dOiEjy;J_OH)jJZr}374W>q($YNTB^UCsOEa*mE6?3=Xq|4DFnVPC zvxWb2<4U+9($NtGw8pBf#owrZ#Lgn!ESl5lTwe*H;L^-Eu#K>nPhB|6tb$ekb)jxw zydmg*nGW`mK-Y0fTh|~@kxwdO;RQA`x}_b^*~v*V%VVdIJll?r;T=UPB+N!;O93u^ zr@W{AAlfC$RzzC)e-R2l<#8e=cS2Bs4z#L{Qkc{mwG71R`5(3`aZHy#;n3S17j`=u zhdrK=Jfp{dA-NqBb5DkF)8c+xD{Jg3$Q86#!= z5wUr-h~LESPEQrSEy%S*65kW+_d~nKktu^225%(H1?j*JNqy`9K~x(_I`eV;e@BIyv&SZ_NhS5NZ!ln&E;6Pj>#sfQKyt;2c1*$lssHSZhs4E zDBfoR-Li-xbh9|e5Qqzmv4_T6TqTGC_v`9wt^JB*=z2O!kj#q$-RVgqc_q;7@w#A_ zv#W%txB@n`KbX@;-5b}xeQ}qd+?mG2e?*a6i*C*4IRL^ep41Jyb`)Kkq_mfjc}k~1 zw-w7TbKuDWiv6P>G3QcMsOw6Lck;UW%m)^TgtAoRZXv~&-Yiw4=imZGeGLCT14846XcB#K9u0lYO2dS?m&rft zHX_|Dh~CPw&1yTUfFv81lqJ1gI+d`ZO8_r&Grn{eY zViAp3`y^~V^<)sUj>Gq0+*YTX&~A?(L}zBBXHN%M_ml$R$pu~dd@96gB!5ujPla}0 zFFeA~O1}warQ<1sM@}Q?{#@;f6;6jTXAwiN+wP2ZK~A#7d(g4-83oOx&>S0{4NY1d zQ=(l#GZ@t~pVwwQ7tE#BbJ{4$93{k!*@+yhZiI_ z2sa;luvm~eC+o)r#phUn$WC>!3qdH*6x4eabam1@aKwK~kOvLc0L+YO=KE174ag;W zEPoMUmw3QF16IEngieA}`BH2ng5t9TtFXt5bQZj7gep^4oGl2JA|~gS=TcF%4M96f z_MGlz>07x3JWm8{!rG%a#;b%2cNtOPP-h)Wn-U*)<#?b0DlHqy)RT4ZqUH3 zG44tTv$Kk8X$mE#vHcqz#IVWUutZ5B*0OSRxA&$PY_(Eq>=}5mO$Y1Dcsv6%-1IS%~ z#v64vB!!#Y-yH)A*X9&zPGM@VM;(*RVammBJP{#zt~FAW-gx&W6$#%EeT ze-q>(>4)z~V~b~sXj*wCs60^+d?`1^rS*T=4fF3{yi$i)KGDv_)=-eELqj4RjwhZ4 zagB9hRF2i2E3%OimEHh2UyxhYUT{^%yX)~7-Moj+%UXi+ge0k%Z)Di9{%{SrB1{uW@Z#jUi&RD&kcHOD4r4Rn$Z5z=w9aS;;UAs zaXMo)7Jdhe65$su{-toP0*BP3Z8PqFV88R*Tla*!+PnNi8?;Ft{MYxK49dT;)BA*- zG4{T)V3_>_5Z9p(V~Q4OdB=hR-9?$Clj8ePf+&pk5hp8b-Z6>O^;Z25fc)hBIL3*{ zm8D0;ZlZJ1*&*W%xi_Z%E{?S3M}0&b4|8tbV;_<5pQa+gdqt5tzMVUxcRGa2>nTi~ z^OjElDO?F@_spOA+ky1{83b3X~JV+Cw$VeOAQdVhR!0pa3$ZA*$F7 zLbwV#SRrfR+0@U$Tx8AzG5m`nO0|+6qOQhYg;A__YWxp{UGAa_MZ=fHN0h{i_nyLB z;>P|m+t>XCxchNvFQbCN%T1CPKcYWiG`6_OEanN(6ghd~8unF@3^vbfCeIMSznI1L z#@ixX8XkaQQkUiM{NMPKNhMs_>lws3z6EeMun8_cTm_jE1Kr)_q`0=tcVJYr6+Njg zh2aV;Qo~%G|HAAvu2I;Z_V?uf&-)&%HVWmK`U?Rbz_?q{iAwq~OZ-rerhBz032^!K zQY8xw^VWHFu+-6l30YrjXKejb@o77E*G@aE zNxu{jbBgYn;y6L>;@ly#EQxm2usGq6-Q6Bjeg$*YIxv{yrLNxqoN*Zq9uByx2w}40 z9qubSH2P1DUieRvCfJ0ughbVH+;t4e$z!X?x~kH4Bpvw>(}v5>u`Fl8!DjvZ$&BI;#w zR0Hmu1B7;G>m{dI&x_2%ZcyTZHwB2tl_|0GoP;So+tX3eBf<=)Jjv7MG(ov>xkQ%# zyh@bk0we>BHKf|uEYcZBznm9x1h`@_aAaIukHFOPprb+AVYj!31;@qxhHA^L}gqG!J1%Vbl|hdk^j z(Cn2_D(n7qAuf|vXZAsOuAU!^=1iv7an1roj0&=&)cL;^<}t#A+h-3;vG9VRp4ez< zj(3RnNN(yF?kUPR}Iy3y#}<$Vg(`z5|y3nk05zyS5V;gzHgLJmrovqc@(1#B)^Zb z#p1*uqei{lu&W3OEu$)Sx zZ6hIGEZBC(XChrr1{&`6en*S9yD!K86jJ^UW6>2LoC6XxWPPl_e-h?$X-|m|gKV1x z(slb$()&SSXV}RzJ!`KBnG&Qh4~xjfYimRkB^Fx=#N}y`oObq=ix@q0XU5w!A*79O zzgT=#!t_w~3W#BM5ara^W)H&Et%f^pBNwEvE9`$chUqeXjS(Q`F{g8sJPr`xy7UdO zvBH5+Ydj&)ou#E4Z^c#&TxGQ)s?H5|>wc=hq6Z<8U3PT<*QST1EdC*gwdNKHRTq|uU&n#D98L7hm~(tVOQI78i1r+!o2N9rF=ZSl*riCK z=IEUO;PP|k;eCg`=o94Zbl7H3-4%}r6F-~UV!eq)8d!iciiQMG6WOA~q_rX3O_+$= z=w!zVF+VsR=#eXptOwQs&5YBVd8G4}pggT0CPCX*d&U{oEl`9U>=Dvk$7(`dzkJt` z{1_c9n4(B0H;(=@gp0z~uQbT%ObFG$%X4(CN7&WuY~qZrz57uc_;WpX5jTl2i~CzB z-X>T(nCj z7nn(Syqhp**;4TuL=H!74qEFO7$l>j_N1Wvao42ogg7H~Mz~9FjXAdj@^Gl49WiyJeY$ecRq^~z7 zi*m=tXoLH-7q$X$xsV~SPl(oS`~lKY9)3Jnlyl`mBiZ>~ylwFbZ;|NUVUg`YoU!hE zq@7ufLv{dk;W(VfIU=1i{hzTi-;Mz066X)JU`S}n&R~{j>K;N~b2N3=ljIA+T1GkDJJa&soJ&o$4v@nURBc5sLSpRLh|V4^~ISYh>F{nxK&}N(u?P@ z&vHCAxuavTefZB+L*b}BR^Asn_mJe_S99BQjeu@#y^Yt=He?l_9h$)1Rm!{!Fc^mj za34m>b(tvV%MLc46X~RpMq!HDj~3!_6~N4FwBo4qLd_tw@8~t+0tH>Tc3o_o*aq!h zlT09x(y|7eE;M&oxW+zJ0E}So(G$ZuAk5WCTPOvj| zISs`7JaZz#g_Q3A3VBd>f#esrKNsT4q8Ak>_CUH&_^#xORVM>D)20&g+HFPCKs+24 zKPc=nCV^#hHOBT9(b0+IrcAVx$0R526$z6@tr5>F=)(2cx?+44#EfpP^rGse&Cr^C zfO!-Tq6)#h9_vm4%Ok`-hha}gR0YF7ROJU^g#i;BpE_fd2p1H6y>zpCsUT<1>o}7* zs7@2&>^b^KJdXZ7+#Z!)P@h!cq6&p zEG%E_QVj-bc2)mX{c~ghEv?n49#JHyS96%V&JmJ(Sz5QsG53)`6qqYN&X4S(MrA-d z%5*pLg8+ZT_~I&{BUU)7NI9v`U>j9O%Cd_^A@S(qv+5ugpu8dU%rQWDmXW(3y*Q8Z z(y{f3Y1!aK-Q$7GMfRGgcl`YX0Jk4I!=_i%eRH2EdfYLtN zPsXhx%u+_M#&35la%zDgsjW!b7fT3p36PMo#H=iUSp^NW*z7dIq==xo6ly|@*_rOht$OwiIs$EK4V9HjKWTa zCygjqN8BpNy>_6j53z{MX}&xg(8cGHgK@OzIRGAabh);e@!TRIrjkCr7B)f<`7lw( z2KB%DFg}$s!a0BEZ`X-yTU|bUc0LH)&szO6UH}PK*n5{^B@rZsGECflp~LI};=zk* zk+`Zp=wgUm%GM5ilU*!;$rq{qB?^*SW2v<(=D!5O?c6YKWE>*WeB$Aj^aL3l?oOdD zpezZHNImyEq=|QPO7WTy^AGd-F5NJEAt2qH} zBsi*fJ@-`DWy0({-p_w^AiuitO8z1@HhH~q)zuL0?Z_OYhhB}{t|`9dMTah`%lOXQQ$et^;!g>3Lcrkp`3a#(?Q5P&OQcLMeQ@ocSyhT<*5Tfj1N>Nb!Y@^3aWrt4qANScDmZ=i#_tWG+}+ zRqiROvHVS7W-@Mjx?+rI$H6LDGTw6Z3v=Olp^lNPW92|SA{Uh~k@5ASF!v*g`6PsW zQ;_SZ#e-UpycvWR>+PNzyD043Boj$3dW^8lPN<++**)wInXWAJ-Rd$AJ?gg<739#{Ga69P4T!Wmq1rT49SMuL0nEYd=ehw zTJMfJhH`0|RJX5Vy!(`$D;c$4lIp z)Ajx>%q8M(Q7_+}csB@rf(KdKVvBnr$U?pvZ@kZcFtD5AIT4;p%H7f-d|rT?lEVmd z9P*xcU!YU3GLh4etK458jW`xUPIIylH*Gfyv$kz-34)<=4c8SHJOJga*yxXEKlC7g zv$C$AmeFqn=EkMN(;cpPsEAhLWQPESa^E1({HABggIG(<`42Ev6943@C`{I@NnFm1 z7L_N=ejTq8P+J$dB+(#b96k4N5f7mNqP5QjxWjOL9VV;=*|=Nh(tC|@=TND|KlGAav@vV8Z|UO5~3c*;cK2J2(xyyHpS)& zx}!-D6ubP6mOA!{qWMgUo1f<}j*#N8e}kL7+-phEK!EFJ=IT^`+EYd95`@LCqUoTD z5$pU1A~(CN$2yE;M#>Z#=YkWSCX)ISPdjqs@LwqB%DWcG3I6m9fH^`hVMpR#xVm5` z)ZfDfo6)+HAom_tye2fzdET^+NhOjFg&Trri%&HcWdgVCb44cXsPyCreWw!V33f-( za~cOfUyu!GCZG2LBt0Oz*5PohdG0x(xfB@N<65vBUkS@I6td{`UW9}#^)5On#I<35 z#=xvU?tcl0W|3#}bzUV5Q`y|MMxO{bgRDT?xVXd+PN3N~AcgnX`85Ezs%x8a{8N*T2Df$ zrQZT^MX(i3r;VKiDM8*2?o&7~J5sE9Tr{O*d)?9(U)Mj@%CFJ?ogf!n%DkD?u=KFE z`NI`1m6@7i@Errq*r}(>;I9R^a9T9kobLM`h?|ButT8m#Jnw_Kz`W?51SBH_d1NDI zLGGQVy%|0Lr495FZ0u7IV<*IwAM%eYg^QUM)6SSzYju16m}0;J?9XD%M}%GFacjor zqFfDBN$H6@3Gl~KuAyna7vd5kC+a{l5NCc2?**}AF9ax?QO8wk{#Lu4>U=wGN zM(%?U$9)0le9`-q#Z5UL7UnUBBj)(#OGuhvhzsy7zak7D^h#5_B7#5I631p=6LyZp zdxEjQK=UQ39U{8wRDh#dzu z=Wo{mB*d)W9|~}3yLc1qfZrf<9rK$ehYE5-ryA@qJ91()OcIOq=$POFDUhe`b;A^xIa zW2hA(vH}|u#7o_y9LFe3`ZU4BI6*{hF0ImbbN1oGlD zaa63ZI25hb(-MseI&KxSLEVyEBFL<$aHdICAg?X~M&Y24YAk7@UwwJRQV>?flOBC< zGHg$Wqa&Agw2JQiZxLaR%hN90m?nhURB^yQ+c3fu&jus&Ws0i+=J*XnR-^u-vDjSmubC<$echrLo7O~0MNYRfOlmb;AWkf zgM6+QZ$SOq0L&c@`bgcoO=hgd~SRZUFPiU#}=Ic zLgGAn7<(w}#3lQWPejvlTv@cnyb~cvms0@kahWJr1I+`3K2HlE9jV!GvL<0?T4gJ? zxkxuJ+H|_v&}n2ys2h+T*{UAJHUC<6}_iq%G3tj4xSH?Gl*bqw(0DqhtfoTDd zx0u&6Aq||^Y-9d$Z>!AgS*XUv zc_N%OrcjJ02AcgE}g_Q z&y2|Q`=o5a(dWc{L0v%P5K?5jRtT9U^T#_xIF;%EdUP?aRHQ;Lk`27(vA2X$wpO%) zk7_JFkwVpA{8xZ`4W@i?RuiNvJYiJ4AOVb;gJZ5TAVo-5(6@`o{av@`7=H+H zAL*05FGJvoN)d_GL}&lonn9dvJKAb@iF8He|Bw3@0Zs%RWM=V7R9lKfI5mqGrwM_d z%&@=I|3ti~;WMGNND>a?2XuTJ09@dzDMqvtrk2U&V_YT5jgG{#D?S%Z%t?eF>>%v= zC8L6Bd@979P2SP8fsBPa!Calr?$+2zq!a8xe<^P7g3e5x7|V7SksDcv$B4|GP%npe z#55r=m}}okY}5l$>nY>MkBD<81DG{>bz2M4AA9}5U_VRuY9jo9b+8kQe(njYKvYQ2 z8g9}@*x4cZm{O`lmyle2mA0ebu}9%M`dWqEEjfE*E}4AE%r)@4G_WDTkVA;m;NGSy zJ`qh7QuFUMwTPbf#??4d2n>U3}iGcgw1AKH_w-E~ybM7)Mz8ZO-ea<6{?Bqiym(>{8T z!)+|Zk13p&Vh)>r7IKSPNA>|x**AIXdWBe;C&f3g0^O^Rz=Juv+bLGBjJ47jJ7=Lmq@ExAQ<4 zY8`YO>iCNv3y}8+6(%7`oZ-lJg54tWNg96_O&^d#P;WFH2LYK7-`HKW z^OW^79sVF8xhHUi$MhiQ({~6ZjRrdIJN0j*(@sTx6yib1v^UT3P%cd?9*1M@6QJ|3 zz$k(84hbO!k837o&UO;C8$nYPPcFs6g7UZ?6YZkR%qn+*w~BO%+&W0?apB1zxy?FC zXp4R%z)`q!)Z04?ol+n;@zi$bOd-ye!ILg+_B##8wNF=aez0cI(~F4gmy-jG(wRW+ z6Ygj*dVzDmT=BH|zFM^7AQzx}vMV_LTre2fgsoon?}VK{0=}erceo&r21z{nc;%Vh z-FblU!s;bWqH4L|TlLlRq3MYn1$j>R0*K6O)ZQr6dl!P3Gsp~C@;AN+z`3G+lk6Km z5M=h!tKyc6i%(E$;k32KCH`MZnW!y1Dv}&g1Zs~@MC68Qi0?&FIF#LzmD4Yxof%Gw zD^2X{+Aalhfp9~`6gqgBzhO&Skt2y)M0>0jX9GMulw+35`6GADnq-4DQ>e4(sdOh> zn6v5-9&k7~m1D6hidfPKZD*mpMi7%s6Wv?E+`llCdQ_x~NN+^0G3iQxJV`Y(rjkC` z@mB#lWBE|h;JHnZTb$|FZ&l27HJH3P8#m72*)pMWe_erbj;wIt?Tt@`(et{v44C73!tN&Y30($V zUq`s>kudxoA;hO&#m5S}&vgtk9UOE6G(DhU;^>KSjtKv$7rW!!{AUYq`@}OM+-4YJ zVKSPOeV-Sqw2`d;^+v+3BeM-VaRl~)oMH=4cf|TP<>=f}E`GDaQ_(E>Rbg7NZF0KhFCsqMR5f0{YVjcNIXXT;ft|zi9w2qhA53#^r)spjL!7&xmyEC$CoA z7uCN(KaH-7Pv^UfNV$M3-vdGc8+9YFwFr-htSHYn+X*z|5vok#l8pnTFwYO;Cq~1) zP+q`_8*u7=w_q1gHkI#-%mw78AHB*|?*nmmop`0Ou%P??0+j7~J07F@qDNq!0sG4X z`26V=;>7Xp6MYXsnm=sQ(drqDeg6UEa%V3j)i_m{8C@Znu2|qv0IJ`^SwB7#k*klG z!8@Ms1Zr+aM$z;bVHcx{tp}a8SBMjwf}{*l{XYRnkcpD##6&njHd);FYvrE#Ux4l? zL_%8thaKG9Jzk*9vNb_oLY+W`Ud2da^L+L1qm~eur_p!%6C~g%D7gT`zl+cE9dg8pNpy3Hd4e_?ZR9LP- zV$uIVm>ozb*)}u?aMxhRfjJX(V>W3M>;#eYHN-=G+a$(HbUT@&uEztEx z52+Gsy#Y;mkvYaT3K#7?A=*Tf4TdIdal)GrZdu9BxiN@O1esSd-9-qu>02OL6(+?c z3cB(wsDD2o($z?xBh!vAg^@SLUa{uegq=5c)QyF)Xs0}YL`_TmP9Ffd-0au7@M^-Fioz%=kHW`e3j32%8T%G`@ogUh ziOo7CZ~PI2ha#H+q#ayH6K7Mw9mmYfV@8gModm!z zG+eu4wr?PErKBV>HTs08H`oK?mfu2>T=JYiMPB*NG`%Cu>DBLx1eht+wH_5hd(k!G zONGs$o)T{`ql?!Ttv>*&0LU`p2NA^9ZheWL9Ok4N!+s&`dLpClkB>#WMR+b9;M zJ$@}P?EksX=R9@uZw1CptS)BIftq2>TCYGMP{+k|{yuglV6(fUdk`tXV7w@Zq9HUM z74L}1MW8dqtTRJqABpCy7v& zwDV&*+{m0h^IYK0jfuD|8A{wMEVH^akc{E~HaC!SZ0_b}3>C;~#5{mx!2Sl2sR%M? zrA~J#tZijS{HS2A9#_4+an!s3wGqdT6agm{{F_i`shhY)=K8k;`Ah8#tMOGm4(|ac zo0`4nBTi-nvQn&N*zoKp&|5(~eZ4q8l=}rL{^+7C0PQh^`HuXZZn>c2r}-yZMAgd2 z2B{S-rmh&fFlc5;@@Y|x$$}hF+Z$exsHUdps=KtZAbXja)OK zY$uQs^|YdBz2ssLW-zBdT-Yu1M*zw$#p#a}CMGQ6YJ71Br%fNld7S7x522e!^!6n{ zJeAkZ8sf$=g4J)6wrD&rk_>PY5tEiI(qO{oqu6;VM`p6Z15}hmI#ZsL>W)(~ZF5%S zb4%BwqdvX6NJr-_=!x;pGSIo5F(0FhXAdh7};QV`F!bZYw=#jdiR5mn}Izz1nI)z zY6(^0j|4bH1`ls5#DOapU$kQgahFI^OSjhdW`b$FA^s=AS+f&M65ekFxhHdN(}L)M zB6h4&B#4!MT(T-8V;(aW#`wI$p-7&gG8m7GaOoKEoB$3R0Z^-5^7$IC2yz@0w@}Uh zMu7P=AR&{E2sf_==32L)3i0mh#W&=`8RU>Pif`)bpBi!0C{U`0sGuu06+uT*3653RU6I3LbNpR&t~z%5 z9KZ#@rYbq`yvk@OmjsUux``f)_3Bs~Z%C)-(YwYF&kc~I|LLv6{|O{tj3i>%Si;Qr zJyYV3;|NkVx?3q$6XD{shfEGq*f{Mo-e30Ne)r&sgv}pqFtqVJUJz`ylr7E^=|XZ@ zuGNr*XH77t%Qb@38?l83&3f`0R z(vo2F64}mRtgdJ-2NrsLX}5N}Fr7)UgYSx8*M;&I%(iS~{tWF=P?bVlH4a`61fF#w zE;wd=!Y*w)JH;h8@L!n9Mu);A+m7e|rZ`fRS;TRk^|Bf-YzX8AVK%8&c@(rWHroi$ z1*F|rCt%58!d#?*q=&vSw0Xk86A}KY0&;{(V`Ds0ryJpR?6V2~yAG1aFt!?nxH#QN z4^^!pK~$@qdzK3oCI|eYQB>BwOX?|b<0G$TJ8Vi6=gCofT>K{5t*I?3+S{6yF5Jus zrPV+xKgVni;Ebync*!gLc3S|sbZiK;!0jT0?#F#&W1LX`yNcgLW{qh=Tz{?oNCUP6 zfYp*}SE3@yCBt*^U^Hz7?O};UrF%vjE6FKagS+jzFn&*@ONP%B)?@j_FJfRk8Q*OK zS<4qWK&-JXfIs2Q=Sna8z83|-#WwCT7Tk_-W;->Z-hHXLow% zdCHzZxq8TGI&hcH(dY`H9%Z`6;l&g-UI}#es1lZX;#I|>UacH5cHfJzhai$2yLEY1 zn2X=bf$gk)AoC<79fYjj@`_*=3h`o_)`^ApEg&9jqny<^vs6H(uBj*)pdZu(l#@sF zuxp)c$FJN0BApwD;w;0%+gTadJmPH79ZS?{M&zitU9?M+jQAMW4-3lEUu$-X{#gag zY-1BRl>sI9N5h(n#I>u933Ymsq%a4F&V#CUGa=Xgw+PSEC3i~nx+PmcT(?PB{TwPX zXD8iCJF{544U~?{Do}}&L^us;7SY-ZP6BaN*jqKX^>)Tz1UXrjlNfA=bj^CCRIZ+V zmoOb*s_}t>Zg7mVIo$jvz}IgQ9tGgdcHWcg1a+y(-FjRgrU;=3 zW35Zv*HuKoFii?!@79q4d`)mWcXT&#$X51%?eU^0Cz*_UDzRn{2z+SK2ECLClo%^< z<7ECZ3pz_&ozQFM?FGuMiCg@%2YF4HlSI>qYs#2b1<6(K=8`s+?gMbYVTmcl`l20y z^@f@b>j!ZC@!KTDam^l$Lg8j_Yb-W}Afsg>Uf>SMVLcza^+1u*APR~MB>^s>+L`se z8jDRWB4N#f9@hi`E;3s%$y3f0;u4~5S&2(TdknKvXu*ygr#E*9q!2R6yN5nA!RVw(e!VEyLJmi1>)UbylTc&7N8vo-aH~>^X9c)9)fMy%$m07KDKriCAd8Nr zgt%I=yJqZ{V?|*qIH|6wFddL(Pq()>))SE5MXV+^Jb(!9<<-I1S(IymV>P5pI!jCw z=#r&3k>i{L0mN5E$yWyvb}4%+T!k^~uY7O;=^|6jvyC7+tt6E(`Va_bxx%>E`%oxn zsmu4aK3ToqBhXp0igMfjpa2r!NUaC9O|XWIiDgC93Q%p5cKZr~^4F|arMOXm^YDH} ztv|!zfNt-`4y*x|5bfsa>rQv0Hwy6tQblF!rTPbDM1wKuh$4X&yJhKe~v7U9XNxl{iya9u7#3D9TL7 z6rW+_jtu1>0p>AcxmjZ)4nK~6 z&1b!>t*hee{su&a@hKfE98Xx&k{%Y=Rs^}TePg4)y(lvpAwpZ+EZV$~+&8VvKMHdR z#Rp_VlyHd?09`_lpw$F8mR?4;&GaOJ<_7038M{6sBxiuVCQ_M7ymum)=B6nz_elg@ z9tIwF3a1naAXLV{A!ZRmeFlway0gq^1P(4rQ`X*P)3c-+3b4 z?8t5P2tllM4v-7hg4I$CiuPpxXlsH|o##Th2#EAhdY7NK8wI)s zllpPu8KcfC0G+i<(I>$9w<3CzK>jvC=A10uIaT2<`+P8J&gLzKUr?lB8O#&`PJ`!W zl84rb7Xs13ZT&GtK`{+`+HVv#kGXx9j^q{>fp}+??s!;TwB6WSu#=RLjs(E>3CiDM z%RqC?b1?`h5AibD#tNIQrKyO8L^`rMJ0o`YOF&#Ji7)7fWn5wSqcPq-{2~&5w&;ps zhf5*cdS%u~)%AC}JTH<&dD=`PTv+i^D8e>{nk2;POpDlA3$hf>#&ONQ=bA3os*a zd#1DNF+!*U4uRrXh0W%Yo|a~HS?C)7tsT#Z9G2I(uN8IyP>^HVneo~JkY^iiRs}dQ zW-zWQcs@dq1{~J&agm7JuVvEuK>icmbmk8cZXT{dyO5%^MD@A?(`g9{mvX#R$8=?m zeiwSE!pv6gn52!($BSGKSTqD)B_|5VbxpTuy|F<(4rV6Yc0Ey#!~$lEMxw3=UJ>D(QCCN0X78H-TyY++ug15c-B&T2jL|nkdVEV} zS$}N%ulmFBu;-3P>m(O(@!Bo?o4X&K&|1Fm6|Yt_7#Mw5rlW z%0kVx9ttw+G)UJSAL4A;Ce{HgZ&;tU6TpDrX{*>@VTz5zUj)h*iE`bvuFFA@?!&Lo zA7CaBH!JMQurKC7AD;>#LuL#gf?DndaP1I`(QDN`@3{xirNq5ujJ($$u^DJYP=Tmh zfZ04K&ChM`DJWBEa zM6DArym+99PH>=5^Q(&PS$jMvfNyxM2D{x%Xwi@FGl5+-E%adV9dbT)z7+v{$IT17 z8xbxod1GW9FV+!eXnB_(dWbl*Ln{THrm#za4>s;^|C9gZedL(%52xHL=Mfu;@WjEc zlB-Xcd#W(kOjk;HyU{-TJfZI1+?poSOlH!KcqAu>ZDwyQ_b8EQ7w$T>*MCT4(RNcY zd1Pbxvp{Ob{&;lk@EC+k3fJl0o&RzQH09Wle~Q+}i~mv7B8PqfHMY}h?JTtI2~aZ# zch_=TaEuV~U2F0qBIuX4vHCFo#ja;$9QP!Dx!JR7Ykd0@kUNiDG{nfKp>vz!&a4tY z3dj*;;6Cerq0L0ipXs%(#f0Ua&88D(EF;8UA>J4r>poM&V1pxPQ5!r9ViQq&1!ZFL znRBZCsSPDsL<-$>p>v)C_AqGDeYZ%uKxwEgR(#%JULlV03cHTIcw18&Z}9>U>}f_@ zIPQKC!e!+l6e-@_C&Xn%H>w;@h&BrljoG;o@Bn#4I^QD5A!HTiezs;kRTK7rx9Nt{@VNcv__U6I=Hr*}_2R6~H`& z`nl*Nhu>MS<7%(K5Mi=ls3_GW>g0g7Q(>?*{`^>k}XzncKMBYnK3Cw>-1;+<`b+y&o) zaDkZZcw&g5SB{m0(y(~ijopiIQ+%T^pYqgCU(EClgn6aMO8y|Sus6;IxT%dc(?9O6 zI2hF1oILa?%=KWeEd!rpg}9lLyBAvHEI}^!#4)k>yQcJnKzl4FiY@EdHDWIX{bf~4 zINfc3LAfz{xwfKTRO8s3KqCUvmKggUQF32q@7*RccO;Zju*He}1(}bX7{vF*bpmRm z0=GfTI_kV@y74*A$grg`pt5oUM?txPBTvW^U~ z?z0t|Z+ekvD>h)LX$tU1lv7GcPu8PIJ$XcYD}Zt%)$NVhK82u?4ZLeyQP@3efM;>> zW~j|y^JkzmX|k-2k3>0rDI~H1?)@Bu0izoqNl(reP4YOk?v5`-xY9bxlwzqb9h;6l zrLCA= z5mCo-UR>iSlfYakH01D~!UYhif_PHGc&2T!SdRFU}Y3DHBJW3=9@DG&p0zPvEdr zlX`qD!Wk!P>X`Xw|I36}iN!>?6ll9}i0g}K`G0opJh&jjS@NO_F7oT~H5X~1KbPYt zt*W;~WOiUIolbRE{aWO~H&mn=(*#k|QLLBO{YE&m7gd*z7(T;VwSm9`ezD|?kp6!X zCjVUN2=M-#Lmsm``<2v{{x~y#xU==(_Mc^-NeG;vCy$H|M0m29FfOLd0_Bo2@=INM zE#{P2!7_u{_eRrf&`waoC^oA%2%+GO+)J%KM-dyjel_+I?OulaJ$k3wZ1y<;&EP&x z0NN(cDJZu{xx_Wah60>zwUtfX-lCn5{MMor#B6roTtz}HSm@|*al<@CJTBLhO&01B z0zHlBG3r(3g(Ph@1j#~Y9no-vD)+_SBB)I(b+0JwQlVN-FF)W904}+l%{%H)Xx_}8DZ|sZXL@?iXw8V1? z=7UdmI+z6Ik1Yu9#F(3V(!GaUyb0L1&7!Xv05(cqvJC z8F}Bew_vA77l~s7qn}~Wu2?(W=~t1pxv8(Ms-vBz`^}aGryP3r`0N~>gx%}O%YnPyu%U%DUHI3J zS(gWO$}5C@mkp)MAb>%~~TQY0OqwUq};2-ao?HtB;gaa9m!Iawt=RoM@S1^k`(6dMz=d|5^UVk`ke48e*+ZL+>Q5@Ng zaFRPDlTrs^jQ=&0+KqA|yhx-+g3i)3S=uy?TLaj|kqQk`r@=^=v*K!6rySlyAsWN1 z)CK>35#c%|-&xT)vH+&E$?xw&0+{3x5z_155JfrlEENxmFVFz#PR5xkIen!|E-I8R z*~xf2Phn?}+^@x)qYFS9fsL`KfLfa&9ZJ2rQ6Pmx9lte}9s}WuHOc{^&b0dpbIx3? zu(L^Gsu&CE+;nst5sl-VPGgDAKTQPX;_1CuWP<k!|gJYk`pg9@(S3Heu4N4kF6wj82hGQx2r#cG0j^PH8zk ztOKQ~Py2;+MYuGbO*l}^|52fd@sSA60u%+)&o&u-m1FjGom`U1#BHKTA!|hcvK}PG z(lh2G)+g)^k+qDYOOTnSMFce!0p=~m)7u?V?0Z9~D=bwi+PfU*ZUE*H%i^VN+0Gk+ zP?K^u#C{^g4^FsmC=3IV0xZki!W%(56ZYQidP{&C(T#+1qY+?pk!#Jkq5c_%A*|0=*o6PZaxGAsig4>p()HnGqWy8h z__6VZNSBJ^if#)wb$mULqYiY7Xex@k&scaf!g;A6G78E=bp*u*y@C7W=EU*NsdFHn z5=BvvoVCUuwt&bpG&UA`kQl3{Egd7hCHt)i7mnBaVwtTVonLw{wkzHf;+8^!)YmE} zqHAwc#F923b=X~qWAOqHcTr3MnJ^Iw&6n z*uc>6*9wFmm~6!o+ZSJP_aP5)RA>`{W_Y?f+^bGEw{ds=>F++0Nlu#JPT8UO2rKP8xQDa^3!xF!~Rd<7_PF+6{_Iv-gcP6sFQry^1|W^#6TvCd{s0FU|m4F#iM&dk~77Knf4-F zYYqG?bE~}X?;-4k;QQFUiB%imCDdkS+Q&&$@Mp1%!g*LAoED|3$V&WD5yc#gheh3; zj@j2SkeODaEW#~XdnZTxqiKR&M|nMrjT#GD?2Z|l3L429-U38ZC9IHRw8E+_M>%$6 z>xeXabbRE#>l-1iGx{ssZsJ6t4CMSeWze>}Xws7k*vN{*dZpxOg|$1!jbt-Hr;=QW zlgc%Ms5`rX!5Gm3K{2!?r^|G=nq)VCJS92eGSsj@}&Jg6zhr$Ktb5x#qtVgIz zCfQ`n-d4~pQj%G05;S=-9k;~iBFt|zyV~WX2KTD%fVuIw3F?k_1ehZgbPL&VJlFw3 zS+k+u%Oa^g-lAi4m%|-&obwbW8g&>bO^t6v)>=2&dN70z>n=dHMfo+>-vs7y#Ziu4 zti^ba9#E>LLzo(UFVT4(OKR>13h*G}bZg7?V}-f5F}XEPb`s($%^k0Zb_r#Sc~}+F zCCuLH=yEIf6`&6D^a-6}1-c*BPt%ncBRI1PRXVsiK?wO{XosEf38LUW2hUQRmjA-t z1FP(>qF^E}`D2BC!Y&rBp5nwQkWLk+x{{U5Hvr=7>9*SVqgUDzy9qV}x^TyqmdJ#e zm#myUSQzT9?}U<@yo7Y?D+#9C~y$VjJzR+(lF z2{WhrTjM=Z?yy$$WfB-FK0=JeO&{yicEU2plY2sK+m5YAWHe|iHWXI~BXuufZ`VIl zrMfZv=Bd8@aug1^mDp{6C>H>shHi!9F+r*{{wUv2*!4umA9q5Z34rHq&FRGdQ~h&o zuq5SQ;lLurq!vY;=ROF?4A)jEZJ}-vmivHj_)3Qqv2Z@&>$FpZxp?VTm^;Yx1?B$C z>c!FH&?1hkv2^lyUXU}SfBBy1iNnAc7d7LIiw-a1Fr<8dd{r3iWUIB~5&YAx$a%?4AC zgLZyQ2D;*m2Lw53nE|IsW7zTjj2QqQtvIGS0b1h|e_MDbIuVM>C4=SzM5Nj@#_#-2i8 zr>4{Rwf;9=hNNQp&-3^@4@jqi%b4?vL>js0V~?@F5TEF1N6tKIT~^4OFDSm2@lVof zSouOQPXmlr%Cw3A^RrtL(|tu#4Z85u{&-VV9;My=O|41ICqBOjlu{$hrUfsC$bGPv z`JgE-76Kh4AVJFXqdu$wfiT|5K`5sr;|DisQlEI|~&3wnZK%(3UZOq zI%Ye0RT0Gw8msYVLC#7dtZPJ?p~;vkZm)lm9vR^N=FO{%Pf!kxm9K%U^$m9H3~`h} z1X45lW1M|$5ua0Sd@Pc`P|l9w*AaFtTHF&$Lim>{CazJ~#b@R!#p|MLV^UKwi{hKY zXgnU<;&nn1t_()r@wrGhx8@zh_3G>MUk~ak%J(|AF9OK28)wBgDC`1p4bsk2GTtq|nf z$c@v{3TrA)7DIRCf6A2NGlk8=Dgv!}Zi6&Kq=8ex!bXT&r3>4j_><_oQc0JI9?E5u zK=LZFC;lXd)k1A|fV<=zQ1SGS@wXt#IG`ttuFQlU3yOzDIH#U|WU&2rI_98s=I#^W zl*(-_X-oIEAeV$in7yX%lnvOAR8#&_h4=$0!Mrchtteq-SG;mRfSQ|Z=X;?3Cx$+KafkpHnGJJ( zdH0M50cjkZYjHq)QIvaI(wEGp=3fbBO<18JW_yUB%h;jkAy@oILF1I4dtK2kVZ+FA z@v=ylkV%L}d8G~@GlsYC2%yNEC1Rb237gM4C`f~I@kiuVh|A*FLZ;Z?TVaFG9Kf!;L4AYQOw1`kkl%~YwTA1}Po&?WLhEp8|o`5{Q z&=k?7^S=bSRwb#5Rb$3~J2j4uv4g^7gZKPu?41d2(7IztgufeQozq+ENYd`F#_d9= z{vzCDN`41$O zZ@^0JJca#zm6vijsQ1PF!kjHznQAmV4V}~D@r~BlUw|XvWN>6WE7~bH1mB|+#?#^&JeBAgsnfgBc5MVKtiEJR`|LEW|WSbF1v6H#w$`8Kig zsFX8l65HqMYFR|?@c1e@_Q2kMQV zg;GCVj-!fWf-A0;I7Ebo5gvkH&0%&5amWYY&Rk-_n?9_QSX4hDvNo8|D2@X@0&u&l z)%6;9Uz}YBd%AnFg!C4H{UM+-%OAh1MFD$6HRolNOwPO!dPR!hIXF#;pbBH6#@4Euv7{gAY|NA1T+Lnw{Ph26`y-<(XrYrV0 zh2@zp+k=*3rXK*^uE};!=4GuHrm}&E;MW#QXgUnR*eoIQNV2e>}z%kL3T-6!@eF^8x`89=$&n zK>J9V*&5&1|HdOOGK)QCSi4p}-sphIoZawbBQ)1T3rabj6+*h6cbIDzpAn$e)w`SY zHui>s{FNazZKmV6MBAmgf!!LVuEFG${Z2uS$?V>* zH)DE8;6H-RNohenEz-5g4rCcn!{z~WF<7S<*>Q&uf2Ha1ZIS*82l33{S`?b+1*H2* zjjmf^7rWHPi2PUnuQH`OrYr2m=t|Z>@BRTG7v3+u#N_#$7IOQzMq$Sm=X9>|BVr{1{v0;~99=2XslbBB4sApQbz}UptrgBwbE^RuB zSzug^&6WXoXBlFPh4Q`tCs--7C6o;9WnnH11`#afE%Aq81=P-+lAx6YIVre7ukR7y znwL5-FR!5B1VT6H=6q99*#9Cg)Q<9$^3&$`(x$-D$$i zK{iq)G=%6RMAXp)a!%6&WuOZ}oe$`DHzhJWeEaQbK|#-de2%Kw|fPLLyO74O(xvBdgd=CY2l2w_KW00Ng$6s*Kp z5&oohF&rNXBSX&KB>GjRH~|p9x*`ANMGhwZvrfwLa@~#iH}e#oL6oB62tlrOQ-347 z!g~d{0^GWG#Zemq0x8Vcl*euts z2xFGp);X{=@=US_=PAvJqL9{y48hfGlMUiPk z!?;*&C&F$Xj@a4YcsGI0nO!;~=dU{hxa~F0IP9G+#NXLaLsffGAlZybPE5+XKv7Yx zyzw^?PLll_%iK!>Jb_4?jG0|$$ftG%H4oB5q@UL5PM%YUd;Y@zUVwEMidd1ucyFA! z8*uKf60^`cay&gb!$iYGmp~Yb=r5-08<&tV$99kgpg2DWcE#jVCWh?|4Xes&r@4tJm!D%q z61MIj$c?S_KV8I37e+peff&0FVHX5b5hiV>*eDBhJ0WP%-f^XUL6lFLGM8#BJ}T)o z3D(Khct)Uel8c07Q2SdwB2M_Ehs$iZY!eallENwSRgo?Cy-tJ#(m92lxB&?e-iCl*50?lS8fZSyAI`HpS_$xR{0JuNKpwAMp{|BtD&4wI|M+Bhyjg5&_f7S}*B@gTd{q6;kU(mmZH znVIg%bWbJ|ba8iCZ1Lb5U~!kl-CY(4`yq?VV$1h?>%D!O=jlJ{yj{2I)G0r8>J-^B zZ$(We<3q`+UONREDb#lphk0-ASc2wi6-Hx=2sj{tKI%0q%k;R zP^5$A9V>ZCw45|E6$CN1wlkS4NR5nc85K7+WAX~G)&=pTAPX(|WI;UY+OG(92{p(d zvkj+!m^q!@jfme1u-G{A$1PhQG$q!ej)adE2@93B_(DN{%E}l`_plm(%g#;G)8gh3>sVF6-*&SHTo?(F4I; zIT$7BY=5*6GR1NVlLrx5%hC)?&WAI>(+)COWnuTO9&qzI9`S3pe#x@lY^w;4!#3 zM`35rd_c?oM1UJ2^Ka^_YKZIp1vqPE7=g!gheKx#6O(Ir={N$!nalANmelc?AdeX7 zz#K&uqvhAlh)l2;a`1?#Ag9I|sHX9WM}m-_{84FRe`ZEAmG)|Vv;9&0-(XX>0KHrL zS#Tx+ZBWnyEpRlDWmIBCzEG1a_P54gj^STw48>9{4i}LL1Jm%qSngN=w-X$Gvj&&h z$wmVCgeU&GqD_Q3FQa*+mJSf&M0j-#1|G}ev-v2*xa0WO^-)A^p5F6*UYMC?_O0mo ztrrBTAtA>&`gjPJfHi--C6fQ7_K7u5fN0!-L2<~mXA8}f^@lTJW5ihwLOoTOGs1d+ zfvgl43Ua-+mAU0U;iR0VQ&eJ8(X^EohOG-uCTwXit}_A11bOdM04)G+;Z->wxKxO9 zK|78^61D(>T!Ib^6!$+ZkC7fyq8!%=%6h02UN4AF`A&>mPbZvBJPqfcgK_H_p!5w| z#3~cw>x{A;?TcsfU&bk`Uz>w1dKO@#3s-pJg*j(sVdfs4ic~GfG-Gkzj~#%{Ergl5 zsq*gobCc#4V>uQ(hyTsW%41^BbD>~eJ}+k|?1F3e{)|YoKz*pCmtu$Wz}#MHNpo=7 z`8iNePnq8*fC{fliTp2|W5xo%160Wi5(OtB1%q@_T_YGF!uL@_Y zfXt!QbOzazE&^8L#l|8o7GXB%Tqu-8=PL}3ebbV1tBXOi4u#*xdh8`YontCJ<(7<~ zU>y95$onf3rdsF+0$u#l}M&{t?3?D|e}VP7!F!}Hn3Hv*BE4o)st*ojpr;8!C3Jv%c^Mw|QtB>#iP#k&d!`WC+@PqDkePNq8^65@Q>+F<;J_=n#MM(tv*TZs!qQ1E(ZSKN|? z%iu@fmnVgfElpip-49}^X^7H1w7no#TGFSbBl}$hl43h1f`>G}85q>#bnUU^19>vN z%=og%*+39qpl*w6vvAA!QSs4(dGfTJPQ9Ws7<2s-(5(vtli0aQ)_5H84w2?49RVNs2Qjg+G2f?{w}FU z7{R&(c@e>5M%7|x<7J1p^#?5r&sfaSOFasebqv|ta;Kw-*oHR#DFo59jGhorDcopN z%;_JAcHN?h)m0IWR|;eJ;^Au=~HplbRjwJnPIZhYg3T>^*TdJm@Gn=twM~NCn0Ak|yP%Qi`VKwjM_*_I*2Dt`; zkt3f2au%{uCeP;q%)?~aknAYhg+zmb4ggar5Kq3m`JipNbO~{y;H*MWw8qsh<{wI# zL04PsB@pL@RU1I3fGq6}J)|AqWl^MTN(!gvU(Qoaqt)Om`6tW`eZ1~_gb)k32mh10 z(uw1M8-=<#Os;nLI{PZGf?0a>1lgXT0C`KGdB8|VBg}}`@&ri^Gt9IB-$-a0C)j*m zD#9fhK+#o;F9oP&P8c6Ay^(*9DTUO0QlsDYX8wr`AG)IaRt})DKzls&Z2&ble67U4 zGJ?sg94EZPKZq9XRJFgoBmO4Z1!;Iys>Fi=U62yKMyROe{srQpyR{fKQT|Ae7Q4mW zAkZJtTBys|?Y$h4_iZ>|?tKs!c3i5Hl?7PhT%>Pn!%UU>KVPT>gndNJ_5p;4g~`mQ zONrz^=I_qfLIkqn95|j(IBStSda9>c-W6m%d4&f3rL43LnFZ*^Rnon`k3~EEcC??f ze+bza*lDBE{4XcWMU!V?M!rXdIAt9jqLko_=Q)8+xx2kskLR18u^^Iv@?rl+44Btx zY0MO1Vb~k*>rH?=bo8i0KgwgsYN0QF_c4H(L(##)lVm0eYp_BAQ`>%8pbJ%KYmF;E zh0I1Mu9L{pY|4M~q*xrfiTs(Ovv~C0wNpjP$D2hapjrZ3kIM6P{>EA#&UB3b9yNI_4b_r3>X^A%&L%s#GT)54gPQEu0 z)My5M+=W^5I{=qKF3C0$Y4(c!B@DOYZ6R=?J5#ab_xa!LdKnWl<^O`19qB)joXQy! z1)H68+fLQAi!hXKBU@ti9|+QwSd_%t3Y$yLI8ztcMUamxSntS-9N#nMCEVQU)RI%B ziefq+7PY(qLbYu5V~(+A?oaELpPN{EY^s3ZNB@+^ODj;uytr7{Z)QCjFEJSY@`JxZyA(D20+%(K zuQJ<)jr2SE{i5zVf-LJcEfteaqZZo<&gv9{gL3TN1mtRkqg(-5Z{W`KVzWb15%sn> zV2=D-+%DoKY?c6O2G`y(-<h3jZ7zUp%#jbvPSKVY8ewLz>8j^Lgt|0Lyj;Db znanj07@}#FEo)tbs}c9p%zSa05OUSye2dOU*n(wlr6-nRmmxsTCp{>K*&40ky9N9a z57x&b3lb)My`&yHECgXTIgYExaZQ?yMIkN{O>X#?i*FS+FBLfeWzoMdh#Q}5H99N( zT>2)vr-fV0NCR84r56EkIi(?BTxOvo4AB-R;|Pf$EHNvV<8l%16wVW8o}Qy$&JHk=2*nDE7sHi-S4kbZ;}+mfbFpytxaG&Ezl$ zGtL=Htv?PHKqj0p#J)k;nb1{`c2tG%FUrzbUpBDl5dk*k?%)#LT>f zD-wpOjAosx?Ni(@+WF%8j(RwKDFC=Cm2pddfe5$BG3*l7S~@3rxHqPYbW>;Ksvz!i ziAnyHCY*WnkVQ)@zf2w*qdn&FLb|GQreI3K8EO2eutmpY=0ovsmIZ4FMt3geTMob# zqN{dl%ohk^Q0JVCwlU}Oc`C_#mo;(h^1J+_qy<6`e3T$3j7=kEE_Vxn6P&xn?^Yme zLCWbp*^X}_JdCZA@q>go0WAS|ZDmD(Y?7#OH8j<7MW9pCZ3(951Dl9uDA|&&L>zKv z2NQo)*fQV%3(Fds)W^c86I`2fr)3ogvMu&wGNICMw<nOZd$0@4DB0d(mbVuF(HME9Aa$f6YeTy#@qn6kDTfF$myC5B5^-UQXQ%xTVa*NgbMGLM2*%*|HsX0(l9YW3HKx-_zE)>#5IX@WxYY?&? z9y*eLoIaPE%l$0+C0(9wH(nFb-VRIUF1@A5VlH9KzI5F zqqKgWPCA<#x&gHF!uZR}r+b7^pS6Be`iE}_;f%7;SPk+2gl3%*LnC_CT)zjQe2njs z?4=^rBUx^Z{zIMw*9p6OF(#x$PYZRs>|peZg*Jj#g>oi`UR{*`O;3ogE7}#)%1S}z zPTg^rK(|#HSwFEcBz$wZ607n}^8b1nh(BwR18mjT*_3}RC|36@e)biRHM?#`xFS1} zH8ull%#qTMY9~b#zfU;%V3<*gwf~rZ$XLWgOncp4Sc6&Cb&Rp)=oiQ=!)EVX18I^|lVG&JjP0b)Mob-P`pzmaoG@iJ48zhPGG;!tzl zEkVq+cGtOwlySn%HAnnxORn0Id>>vA=?3z{SV7R$(|} zV_J`4e}Zr!bqfId>I;OxE5}~R2Kw)!E#d;E1@6*|ZVQ-oM%gaMu>xE|u6(z|8%-K# zS9n_4j{oNyGW)90su0<;tZs=5L}WdXv9D%d97}Ew>JnzpI{Fy&7~!s@LU(c!5qq18~6Vz})W|0MFA+{BbOv?4?v5UfP4MpaWy+x+VLyUbQT)U`2 z=$p(W(}g)Lc0Bm8Y{t>ZpDy>ly<;AShunxGnnc-K)?&S#AS^&R$LAjW&H%0m%vb6i zHGE2em?z!Q{ACwNvr)ysIaT?pyXH}lT%0b_T(N4Y#3iEL`82yVZW=AP8?bAXduNRJ z@`<#UKr@6H61(f3-Sb%3Ot*3GoIbJV9)OgArw96Ch6uBqUUjdnE1<{pN0w2tEY`^Xu|RWdhmc4Q zW8M9JgQk~!bcy+4p$uCr;+UHj-ye!H$obCs3L``g>bv455te&V=KgU@7L#Lvcu`^V zi_QlnORQ1=$;K(vIa-?TE64>+&ZNX>MVLRVaNx!SGPDR{))_UV!jXaH?5#N}4|v>^ z(p0;kP^YLWNjK*g6Xv$XO0zGfigpg%lGCB^uRNl{lmg#EpZeMB`5)NsWpt#T6nfBo=;LLb^wQYe)KO?W1ZjrXA2MA!cpH zdWAS=94C}@<2Gu-;0z^PjOifkJW)r)S0Wuq1~gN51QJ=l{~QyJ*5k3CZ~Bs178* zd1niQ*wNlB?g3$ikfts()=5XDP6#OyNMLb zsb%iimnWgu?qTF$Az5kB3&?)tM?qPy>_?*&zY55z0;71YuHn+44o2m1p*VW`2{)=1 zwHl+A5H}TRlj9Z9u939OPlx!YP6ecKup7EbLC3FYn!axU+AI{9n0Zcp8i3iPbtbLL zN`m~2mZq_aXmiPBA3O3GEzIfQ0)^q{X8~GIF;LXv`RPz*oHut^aK+jOfMly29Ltm| z12i5Q@@~Q%lLHWS^1lipCEb~;#Q7Q3TN}g#W$6R?&#Xwz+sjB9txb$~w3(6+(3_Tb52vaR>p>8g+oBx(pVBU4EtAZqP2e9KiM#14WSlFI^TOvwA;9BgS*Q% z+DitV%i zdN9?yQeku3GHOD6EYfNBRp~?VZ5Fr6lfR1`#h=ax#c%<)0z9aDG#L4`Od1vYE9?UG zXl}&6gAlqF%Jb1N=~xJ}k@g_FH))osBJ|U06lvIeTL^~C>6w}dM7Gkp#!CZVDbzp{l zQV!J0I4_7TPX@5O(7{Cgl>C48ZN(}cw*bsB8^UB5!eB7(X`to?oeV7{mJ#B@7SLA+ z8X?G;<8GGBs{)W7k1G}83K3?PzRPuN)>6Bj4uC16AzpPAZWE~n!)3$w6eiD>bTl{Wd?+`Tu~E1H!tBvFCPkNM z=ggZ-uu>7=@)qbRyxWIS$^F?^y_^_Gr_uShJQdVW%ypt0t$kt)zYx;w4$vdhC5nTE zxdI!i1D1WK359Fq%)L|N(&ZO4X8GXxG2JAK6GT&exGv@$mzyDAlpb2?h)+fNMn}3QVji!gXCO>hM=1^q&Q06=^f*O8h9yRiXvX9JfI_YerKU{H!YkfwVJv z?V>PQpsMSSyYGN-7Rg(DlEs{N0uCRt7zJSf)KO%9K(_f^{FSv3B%nj~L+%DLL-cgc z7wPB2?g1kYEoEbC5!qNSlO@DWZ@oZhCJo-CVJFSIlJ|nS9JOjO{wNZGB3*X>cZqVT zSe45e&)Y)GJkF>v{hp4}ZH4>tXnKT?ZP_1$xJ9yx;@VRJoXvDh9Jk2+m#qU!1Ut1w9))y4ndYhX76BGe zQiibM>x&ZvnyoJGCd6%zL7N3#Z`JEgTRaZrbak+Oph#z&lzr?NguoZqEC%C85zdxF z2>Q}(PXJJY>C>xm?o)&I+WYz}Rlxjp{3;DFuz*kzVuW(lL z+}S`Udzg@{Q>%*qDLU)ajm;Msy`LpgL5uRmGkGF6Vv%170Gwu?dbtUfqvi>N=EMd-? ztLEKpvB4_{(rZ8&vy2z9iD)y+W~7_N6SjBFIL!CBxPARMaeCx)4oj6L zh7&$CmlW~=VcBpth|-D0*kH{02B7O)cis4OT_GwF7A3nYY`L(9<1Xo6o6#|$tuY>c zCd8>?=fRAs9$0)c2l3F^C6`6DnAHTi2#p3q+Q^>;G6o}1wmIdup)3(;vos$)BZvmZ znNn|D_YQ>l!Q_seE$Nl1a+}e#uzb7SeA{nJrrZ*-$WzZ zbUk#uFz1F28;gr4-Uo2)bZG*e^+BGFN+g~AkXb;MXQe!#`Ri&S83XJHlCEjgzroy6 z{QO7SS^OXr$>WkIX8*rD9VrVLB)%7tRUcEe9E0Hd;-fqo_9QZ{pDBbj*P|l#Ra$$+*z%jPIKV7*exTN^95K|_&~?g8c(o%+5C=;yvB&wQJBBu?RYxN2LfQAm&YThtItF_nIgv$ zy2motS3qW&+}YTQraWy7E8bnbT`?~p##Sw^UfAR`D36k7>$PJS*GEPe)aPPzflwU{-9 zz#My59#9nFdPFs!Y@KWIy}(S6D0pJTFVIdm9df8G?N|jkvQw*fns#b4&}sghs)aTXaN?dtk2a1PZwks$@KR5<;uO zZkR1XtTnI6nk+}k(Js;_I_MMO&h`U1&D9^7V7TX}P2~ae0l6?bO4UNV9Ipw4M?6?5FoemUMk@E1t6U$?MD+nTDv!!AR~&5fN59%U67b= z2U1^a`%Hs;ej!j=7j7Z?V%~*Klyru(kqEk$Tu^VWu=(O>tvwDC?PX==51pZeRyF5P zaOXjD$VRi+Ola03nBUVDZ#!XbNt|0ykcR|VU|8ic)DFf!7sg*#D^D6~uu(6Na!CsSBp_rz4dx5!cGk~6lfd=-# zM~)_jn-&*0(hVB8?i!n;^gBlGv|UeVR@;27dA>&xDLLK)@vf8+ESL-NxQMJ7XXikt zjstR0yQQ74OL70OK*y6dQ1(u9j|Xxy>_c_VN@uDNjYsIw_W=Gz;a}UIdkBAEx_k0A;qj{SsxoA&64->#2ba2|HI7db*l(rx5<- zo@mVTdxwkFt{ASc`IT4nnEemXWMfaF!$p}__>!}LQXC}ABH>Om%c;|v0G7LSk8_#; z2RZEe|1N;Rsn{0xD`?L2`XCa0O$ZXT^WS(wWFyO}H$NWUSp&ih1p8^T#NXXveXS4~6N0$cJA*$uFaSyy3K2O7??BjWD@oPQk?z&2J7$881V z%-AQlCl7`^g$Rh;nNnqKbJNz4ma&>x+%Gz7rab&1dzaYvPk9_>ol3kdI_n*Hc<76f z+X6Tv1_>$6Nckv%Zroi37AGA{DV2CcC}qX-04aaJ9h7t9qzbQ%G^1JDgJnYR!55f=#7YH(A=!=6D_Lr#3SiD#8(gkEm zREz26s7@7M+7;ANBnP&np9vt95)XlWt1z|1^$M7Z8M{G}n+EN;NrcCu7M518iFApu z6V-lm@!dgOr&^H4A4RLt>%8@Rd9&|o5{ZbshCzyFg#gZ1ar#2xr{Atm@>RwKU zy*4j3itr~j%m<}KJwN{+mG41&=h+n4Or?X0tA)V_T4}c9B3%4Zzn%e`vJZgE!7URy zP@KIlNVX%vZKSQ?wZ!5EURJ=^cuM%^nxx7>JK=1)%y-825I>C!(Bfnr`~yCm~r!Dsx+TlM*zA zMXkmuM;XGYO~t81IbLl8aHC^o!R}WY#jV=`&1Jn4V`I9@?t+{TuYq@>fZM1e|1$Hk zO8eMSpfhIzF2^OJGpVrfMYZ!>CkX5$E#+pD^S?Q!la6~AA(W2YJr~n{5@klv&Gt&| zHlhp2ZJ5;~W+_Q|^!sj5%R1>(rTksK2Mj6JQ9!PwuoJ>LS?_cPt{VjT6BY!O&JykK zCSkCTrDamXty_V13uo6Idx>;u2Ff)Zx_l$V;$zLgnuRr0%vW_P45KJ=$HZEqNd+yA z-uT;IBob(qS@;Pe%$SV)xp)>U_X0Uh**&t{pfGy~OntH*{Zj~g`IkJaCo^WWpMAiY ztmQ$Z7B32+c}^S|v)2idX&V=ZVsR0!CC)~BV>Qtvt_#DPD-5&kR5A;^<3%=_eljmp z4`MlcP4*2=&0}akk({%xJ29rLD1>&Vr8@fx?xzT_wB-hC5Y2IXCoEgWatnoSqrc*? zsd-d(Pnbnf>njg{Y0SgdWLK;;4GJFRcAMMz>xp!uuBAoLT>@M_7It{|6pfIR1||)x zF*pchmTA4xLVPa7b(*Nw z`yL97{4qLY`+LY=pj>fGvy|Jz>P3Rdwu{r5M-?`o((uFQ+d{GyfE)a@kXZR}e^F#r zI8I?o-=jrOIVOp80*pV>r2IjM<$;fd_DVY8*|!<3q8oa(xL63~z|R<0Z!$uTsN;kq z_{Y_k^b6d-C)qiUbo>%V5ivxB%9-31%n;>FG8(&F0hWSR zWK7Cb_-e-jIwx7u$bnZ}Doi6OtA=`9B?_q^%dYt3IEbv{VJY5N9^7(#o{C&Jv5l_9 zQ^L$IeOV)ayxWA@s!|hnMBfR-q6kv-|D>?LV9c(tYKgf{1ajBHGCh_&2@=j2AKQDP zcuM|f1&z;Yr$Sl+?7XfLoF2!0X-how zY@kNZWXU(=uh6dgw2ssP#RI}fuY&`pw-k1KPTdPJa1ONV-kw^0dHo^5PR6h5(bQ(0 z3&!*@eq;RW^=hoWA`FB#RY82z(O`zoGM1*UAN*peTBxIPYAz+4RdIU}iIBVSKGXKl>3`#w5v}nty z*w?Q!!|R2($Vn$mb1cQ&mp4D_XC1M@3~2M!Tj=fQE~5Z5qP3Qm%44qpa&bw*4rWdk z1~WYP3TvBP3FWMKSwIKDj3!lqS!=;MGF^x{S{U_KZ7;-e3cGzz;t8!MMbeP=SwH@* zptanrG-(9ws+dyjcon!))N7ULF`So#x!gJ#pd0=w#I>4fLN%w(b2TVA7yG;79fcbM zsC-%%t^sguVR6U9cGp5@em6gP?OUlQi%13KuPcmuSVL6eV2H_HI8nrWW53ZO!2oCLh9uts8)ea9P{N$EPzfuf06 z=<1Jk|KW)A)crJpOnEe0F6KC*teiO35VO>u#Hh~vC@-jbmWQJgQ#BGe@gSsq+0 zNL7Lg;d+IUh;-O-s|e?zQ$xne?}a!;rq8m>=Hl>s!7QVeQIq0!ky(M+Z>BeXUdf{1 za!%Fp^L<2d^Foi-N;>RGA+=n17xW zCfbD_^G`6!B-6|H6m~J>SDEp9#fL!L-FY-DoviB=X=TCY3-90P@E(Sym$XieBNepR z(w2f5{WL-5V=~(P?zrL+0JSFhn7&1XrGjoLUK8o0b*(SPJqq2Z8r>#DEEfyo6PBGg zw2Ci8@=4O4@%MI*`ETY33^w9Gk#IzV#;z?LUA!a6iL$ZeD3RfQmN54t<_#7ta*T2I z6l!Z6YCdIfXAj~}Hah3R^=uDv8jb+I*LkO%F%W4+! zsz{4Q*Vs`3JPBa&vFBxN&vhL^X3Q6nG&wEzR1UJOWc*N5*hCS zh^$gr7HF-!@H0SJ&br+diwQ8VZJp^vc4;Bb*t3zuwAizGG_6x`ka~y^3zIRrj^n6u zto|GroQ%V)epiKE!KjR4^z)D~CNtsa6G6k7G%^->fuQq8?E6 zPms%y+GgA@z*3OEJx*xkF zNCK-yLY!M)Xq5ny< zi-B?YEW(XOpBD7(@ue_xfuCrp$@l*-N0PM-?wp1AyV@W}_Bhvw;r|9B-Ih@;algWu z7?=Y3STU^m5fD6-3LV`sUPRU>z0qeb`jbF6kuHo(?-1!m)yZ`RY#ju+a(TlG8j``p5a-I#G@@MK^emm&$saBgjSL5*OoO9Qp}JCIGpG)O7XTr+F;KeqFa);XgT2 zkRy>&Ocdf8WF0o0COwtOE{+i_*|H_RUt!uF9yidqKau^z;Yj@W8UMJTcxH{RFCbme z$!I*7OsKz$}$QPTjnCEN43`RI)U0FeY+gHxc zj879rRGw0d3%@075tYhq1yrK(h#)h=<5A2Ko!`Ciljz*;jg90NHqZ-2BHz&Ex?jxch4{05MrTp;UX-q{S_L4aep{IRoGdy zO2GpwyUq58)aba`h4ruq;5|hSxnb=e2mhiIE2DWfDDWcD~I9 zD7q}xX$a>ofzCWFr0Ck?<^VFUxFjsbKBAo`w$$jK7z@9eGskKL&IwRz#X@s|S@3cn zI3iAJ0(#D?9<$F)47aajSXD`WCgz?ePm2Oc`8yc%G_l$sI&W!lwiagoQE=9H%}fnI zpflw{K2yUJ0y1f`CDb|GO7rF!aJqtxJl{Pl%u{j0o*5ajRuoQm@9RX@^^J&364DGT zJs)&q>B<~4-OBCD1!r?q?z}HX&JXIATE{YUdXrS=dsn0Ip z&V-YESu0~sFH8w(U-m*Xf{uUE9r$4j5h1HZSWSvkJO&CCZ#1e?6n0Jn8XsG-oV65Nu=jgeh0Ex#zZt9URS_#7#~$@g>drJvo_<$|)w zq@a_Xcu|POpH^bcZ!#mp*myBwz%g&g#DNOCrdfIP#PgyVrN)koRfiFDZl$aq{v1H& zNam6Qv8Mpbo4ab9t?G1qsU-m2T2du-GJtT-u#;fdJkLaYF3PE60Gw1nt1p>D=`I0D z{UFH2Lm!nKsw@TK9FtzX4LD8}m<^7cj@;KIyRrTFT%@^X>&754*Khzhu?xzfqRg)C zh3OBdVRjMh+Lnzg7T7w+oO@|dvyH7lk8HqK66EBuOc*^PUJ_t#`g(h!V;M*vZOQD> zHLP1`8!gKpxtx6HMPt83u(QyhAZPLiF9+l-P>f-WeqIyMMc!ImC%|c8szt}aoS!-R zy5)i0&N+KxvqSn5eg|guYW-8=6OpurI{W|gRv_FMX|#yOTUp#9&~=~--K4tliXi3? zB~f}(@-ab7*BJtWZl-?jIVfo|V=DrdVD@^Xz4w9I6-0?FQpcK?szir+8IiTAJ0FZ>gp)@(OWlnUKmwyjjAOgs-u#>Sn;TJTN9xOzZEajtcXt&NjjLbvqK^~ysR}i=AvXh{vevstec086?C4oPTt%(dJKT` z?5&oy(K}-tNY+|764Em3YC(?FS{*1>2cu&=2rTHmKL)5*MOsGM{KPxIrTcp04-@#` z0$^>W`|IkKTMLDmjqvO#!sTJq88aeo z7XT;hmg<@(?-T8u(TK_XRkn7!Oajfu2{*x1^aMM99i{21xQ`XnCBxNyTZ~-~+Wbt{ zHV<1nA&rkJIDak09qR)j9n?j-F#C>3H+YUxX|B{DEi_o0p^3*yF2FF5ZbvW2O?D5u7aFvOUvkZK_u-|u2<&%Jz-?l-^W9hS7ihm zkM?*{gn7dNh&>k;6$|_UtkEXY^u6syAkK|xDk(dj-57}EwdL7<6T;?}^=tpMI7Wb3 z>8wwU+eD`RuhsYPO`+Ufc|RG|iB_kd3v~HfCbENEVl!y-kZc^dBPqn3`k6jzcX?qj z(ThK&y28$LvaZ866TnAC6;0-IS!B!{V)j208G-2BEY=Z`4M@o)ELAvsb4S3&u@WOi zWZj&N38%W8BUQC$eL?@YEOrPim-gRo2?$pc7$JU8I4dkqarnC6@T~yJor9#lcvOUA za7387!(L@;KueURGFBHC2%vPyGPo2oMY>t%Mkz>k8&J}fVN(=DIEgy9q8Ky_w{=VFvu!d$F z0pJkg&roobOpa%YP?OU@QmQ>*7 zxK)5tf|o-4D4JY2IgQzN$|K{;hV$DH06%Ec%CW2nSe>5EAZ`@Z;0U9PU3Z4Ii1AQ3 zol`elt;gO%oxNINEd~TwgdK(HIw5^ch~>=5CpJdt!UU0Ro0KzOW`tf^izj#CpR83U zLpF+d!>(X(($6MGR}XF!Z8`LCbKBPxZrTmdjBpE&J<}Sy=PAi&G{a+GyeG_MO7FmZ zA=<2YAm`9~4R!NJtg|4L-h+99fhOT* zmCFoBOe1yzz4YVUx8_R@nz+*+W`|ael(}`Jge55*Z~Q32|1(J9O%cniy>kc!Xljgm z39=-xJK@$%rvMj;gBK}RF5f3dv|^#|pV>D@Si)n19bX8cHE_B`y==H2R5n8-r;&08 z*caCcrg6258y`<746BS`^;mR&2s6P%gz?Ht0{FL3#0=nd5w2y<$Jqujg8$gWbVCKr zK`AyX5YL^o_jCzXJ*X_v>wSw84w z^+ur-lzkFw=%+>bGxSc$ir^bT=C+Od!#Zz`p=B`l_;yV99u;Xu(T(zQK|CYGAC)nh z<}kPv)7x?kT{ePcusytqaXr&D_!~@aJ#c;vmx>3MT%u)?jSeWJ$>Oy=R>}z74v$G9 zoG&g-b>>)$orGl#Ku!;7HbN{3T3X{$d5=(+9rv9K34JlG`Kf)m$EiY8<>SZ43(bE^ zg|t$fuhaj^?)+AToe8WoTx)(L)ROD&FUDq*A>Bi0iv8{F>E?w%_fs7272~jGBw6D7 z+2y&rh~!dXZ-{T0*iJ~23N>_n^KUtYX9jpih;!x+SZvx2>73I&qGU%fTMuBP)o_f% z{5+>1OEh_Bq$$M~mHbBHT=|n{Jci$@r-GEi3cRc(yl>!@{DH28YF`fSD!C z>2>u#)^0*w+(hc(aQW>fgqd@Cu<1QPmUdS^woCW`?#q)$jVRmmj#yWi+1HD4(IeU- z(Vnmpr#2(-3|mLMCfdoS1p}^4Yk9J0(mIoi<5lZmw7GT^s{8j7b{4W7jG3aH0zE|o z_M<{98<~Kycj%5~raFZRdaQ8@TL#@#c31SSKM86~ZgRm;m+@D-P$$H3xt>yJkM8Ej z3{%W1T%Ho-hLnyllgZm619^)2u3xT>TjNln&RS=;s_`cUnSZnZa(p&y8VEvT$smu4 z+>aHIb(u2i6_Ng;(1t^WO57yG$rieGoKTHrrUS8fF80Lf3O3plo@9RzjWmmck-6V*DvXj$^)x9Fk3YUVr<5AIMjdmw~RoHcteA9L^ z8$I)vJU;VdUwk9l6~j!;PDAU!B@P3$Fb8;TceqHi)rL7QehLLxBIt_MIEEb#l4XS_ zHSGc8Az^MXs&g8uH1yjZ0cwVOLE})-4BBXyi~Vu$kx**I9p%{eD8g{T#d%G1HIa>F zg>2DL_W_?SxKT*Xs&wKtQy8gKnAvYu*t{|V$O>o0qd`1gpgr$HJz}Noc;U($tHVnb z_E)Hqr4nB67!V6euF^EsPLF#9@<}V^bt@iAIIBGN+~qh*fJ=&9#X!t)Tu#$y*_Dbm z=W;KWv}B7M4`#`?F#2sNG82}jv5T62e#~U;kqh?XJ?n zG>S7doS6bEySG!PkmT4l;Rsf7A?AZyoI4f{XrNFO`FF;YG+qx| z<6+Urn~PnQ`13gs$hRBq!A%M~b-b{#|F}nhQ{oo!*tq)KobE#>ADf?-f1rgAt^a%g zl4jxG7sD>dW0$387%AG#M6(elgxIgej?Gvs$)`tAbXLT)wjJ^}0E>Z|O-koQ1d$D% zrwcJ%gg@Zufj!*`0$fcT70Q9)MHhm&96fAs8tSb)SZX;R5#i>G_ zozBF{@#o6{D6mw+@nS}>=$R0Y&fp(2La*0q?+GEqC6CUCDziH#KULIc_t3H8u~?{^xx zTt=O&s3g}ia&*i&ldvVh^W!l>q?%^|73l9pxGkgFESWZ|J-S3so9j>%B{i`n4#>j1 zoE@hs?0m7&<4&vGf?VDNH4C?PuqTOsHbIOmQGX(%cv+w;g!Pv+t?vslFD&*@a9y9% zs0;AYCmP0kJ7{KSi^!Ue_Cj_TyUh)t@QPhbT&=J(KzYJr@EC}KxSp_@EOe@URd38= zp)YLJ4FTkWPs7+0@QMJ)8QHsq}_ok z|FC-;#%v~@SD4|@)86QNq0AEc2?pma1i0*6Oe)2mqOC&b+F z2vmnAr4zDm810yTqVOm&+0Tz76WRq zmJxB4fNUv-PNc*;w8>=X6XC97&Lr_}t}81lNcw|Gs>}{8P83abPRUmM?00i4xlh!I{r$oup~>2@JeyHm>fp#=(!cnRn$V^>Hp>Xj*0GNW z4NK!E#<>c+u;`!IW5;4C==e^gdk}rT8W+xjwrpiQZ~viB3v?nZD^NtdCcs}| zE{crONh+MnyQ}n_ogw5KZE=#>H+A=Bk^>a_A6zhEo3t>;LLb` z7QaYov095ggr|yRMYi7;Ce%=)PXyz^imS)>3R-OaJUqO^mylW6Scu1eL}y~tVL2!0 z@yJ(T&X4=AsJQXdjF{pR;pRNOWEWoxAZ=OTBa~gfhHSJ)&QJzpKOz2tYp)pYPZf}L zS}C!S7i45uss(>R#gq!kGv%E5Z0unEBL;JzRo_%8x)!k-^`KvX z%p&`9ZDFz15=N_Hg@j;V{2x@t9nyl|p}+p-8+0#Patv=`-iR?ZtR>7}4kV}8Q5NEE zBMq#Y8}V*wew*3fVHG>uMyWS+p7FHB=N5T66WxwW$* zpr)Ri(;qV7a+78BTu^Q?U3l%S$IAjpY0Q{Waq8R-kDfRx9#EM4x;ThK8!yT-w!$V{D-LI05vLdZ0F9Q6TCqU0;tqM45T~P+(9RZQ1`QJ}{J?CCc4ust$+KzULQ#u-M%> zE#_T}aF&B?q+n?cK`y?|)O1jaE}d2FCi=s@h{>{tJ$7HLzBriqXQf4_K1BfhL`}0NQc@;P)juvRx_8d` z;|w9rpl2%jQjz8cGn46s_)37q&EQyxt(I_n_6r#K#dacLmX+`5xK>0qj=@;cC*Pk% zpdE4|w`87}wij{9QqXRgGUP3kZECXga3@OX3bBm{H3t+jlNGkCnPJllE6XgMLmZ@Y zldcx03o^?%$-st2bXI7zRk-P2V;K;avWI*=*_7UggO^_J1WG70?Y_?ud|nXmj@vay&oL2MELJ9 zzS6hU3LxZymWGjQTTw1OI*U?`wz8KXv!%zdV&CiowIr^cteAa{uJp*kdDXI z2GL0+-X;PO_x0n>aFr~~8Ps1C&Q>ohiBN=VP5fV>E|#1Y4aQ=t`V&^MF-&1Igok=` zs;fh1HC*qJmP3Zqv4>!%f!#e9;@=ejb9(tH=34_I8-h?0?j#cSFn5cS6}Ajy^Po2Y z?h)j=lQN4@sTK#U36@QUwn1bdzf+(y#qlIcvj+wEOZH>kj3%X6X)Q1q!7j2AYp)G~ z{LvyZWmiS{d(Nu4jF$l7a+9Dlb{fIIS>^KnCrKUFM7VFH&a?hH#K{E8we6aAW4Nyn zn(>96XnGtcz}$Cnb(3CziO|980-9Oro6y;o<3wQ|4${8bN<@5>30GBfG8>1EgrE?3 zx5pdQnW9`F$tEZ5hU)Qw;EV@OyK1pOOAavVp_Usa!1-4=c9BcASh^Xhos(gFsS9xG z4E$1^FcX&JW5I3}Xw|CGIT|_}Guff&p287Lh$g~%sS>{*ljq2(C6Zx>xM>q(b&FtF zIe#vU_Q4gZD2>g(VdLYAA1??)0PI9*`LBplW5>MwpmBsPXiTrwX;8s^F3_UT;}|1i z)$#fFjL3(HLbiHGCqA73;R>aX>foV+MdwdKoossUCp}y;|3pB`k9k%XsY}r>%xwl& zgL*{jcp;WlT1VH{gU&`gI-gF-&tD%XYebx1(krwM_)4fFavqYdlHa~T9+7PmR|H!% z>pw5lovQ=a6R{zrGi;fN_TpF36pz8Q9-I8Z1X|K+>?OhOJC#8fTpAP!^U8Zf>fqr>MH)u6H0BR=c(dtQCY8IcEx_=Fj*VjzIaJUUS51z)pZA9 z_N_pjHxnkx8~pMJa!zfG-1G-b-h^eXGTDER*cyOBq8q`hw+R2+E^CRnTmTipmEc%@ z8^SIM3yw0r6OI+)=#+#r)GGy8DmX~#^mWotnz5wj>WdZr_i#aG~E3n zq)|LQ%LfisydmCRhN$w-ZkH3je3nJZx|D4C4j0x&4 zBH^fo_gIcp7@-eHfw0XE{)4<@ABD*S|Bh}wc}cXCY4AhE>G zj@g9>E>_r*qrFLqsom`}Ld{ZvD-5_ktjDv$+&T+5w3F%nds#wN>`*S)g}==)X2BBn zn@xn#N_GqH3xUPaJQe<_!cGn|)8xYCv|WKH4y#MvRue%HTJdY0I@s9V02&iYW-39a zo+X%%(j~0%yF<8sJE)2+MY`2sO+v>MO>1MRR_l%_dqBC6J*_B!b&+GnZ!qLWI~>G#R0g8C`hw z4lq}0eUSNXwCKjf!Ojbj)#6HlPLJ&nHeE6+8;oZJyMUb)JH7l&TDaPvGKS-|OPHBbhh-UiNH+)^^m6a)EQS3AHY*UH~(Z+gm7*55X-Bx@NH9!-qChfxr*cTxFVdOf4!j>d(jc$&8~})T#^XWduUQxy z8w?=xw5AoS)9_~r?b_Cbw+ipWopCUbYa3M#1{N{u5GOX7{;(-yli>wOD zW-593WHp;s^%n_u^Wzw#v6g&AsCks_6-jP>Y#uN7Qdx=VLMdt$we4{VH|lgC=^kzr zL~RtSRgU)mPgGVLa>|sh`L`bjNZHYcaF^!zJO)0@87Oq{xQ9U3G{;3+Hl|I$;wJ!` zMV%tCh%O4riil0AtgB?}6h{h&$?j@7UdqBP<0eM&MB^-5kVa+V1un5nUl2y@u!>G4bI|+F3!m+e z(Nq><7ZDa-y-MD3xd8LV&Yat9e}#7Hyb{Be{&-S|Kk3!^UmST30Ia2H^eGYk7h_vY zdp`Uvpym2cO`yaG^(I5`IQ|kaS18J>a`avb z?M!&&QBC|zA^xh)D=TSS|F(&-LPApPp0~X$kB1vPY353ChA__?%$=9cfO1|M>|&!U zppgtNOJl0S@I6_c=57<=h*R)@63>Y?3+drzc!)2Wm{eBje81~T;<` zq(foLQT_#%ydKhQ75f={7PB;Kp3j2R{2!3+s5n&$v+nCMpBCS4i z+)bda@_LmwT1n__K``2Z_CoSm`DSPj*gn=b4=uL1nU3 z<~H;;{xDa%bEBz+qaJ~XrdIBXi?VRb*zxhY!sZMe?)VWg`>8Cxsf3Cb8btr{H)x@b&vJbgMCbwTjQyIFCw z!Y%+Fe)S+4ONoW<0?e8QY@nwZEyUT%&r_Tt+MH`Yl1^Ov?gld}B!%L=E(5;@g}Tu4 zir*ctH39U$w24#hAx73FTW~Td_KygsU4IA7?3^R!| zr$#|E<-}cW9H1~oVr=T?z9%E=Uxb*k9`t&Zc&iDdWm(LBzrSxqf77n8)24xU$6)p! zZ!6>Zlm9dVB~|`?4?tOX104OZbRGI2h)csoYdv0sTLNL0JEKjusT%la{wc<#y4rfK z5X*+kmUwdGakPhl%(oOx7=#OOQe35COQEYFQw72k25eAwipr9~)rlOfz9lHDQ9bp! z&?C?uW3We#KZ+z5ZUbt|-yQ7&oJ3L90Hk_c6Vg$LTJMX3TzKh!sJ0=Gf=~%9VdAPtYh-{DW>Ag4Xp+pC3En_!WpqV(paI7J}& zB&nvmIdpBF)d@`Xd#lt_(^f)Q-m!~Ic`zET<>WhcOdCWd%~0gNg*zu)5pg3gc0zYq_KCT-nbjU`@$ppMXOaszky zm!K$gE0g0j3R@)II3i*vyT{9UTuzF+MVYIXG2>$CS0POn zbWT#s=g^Jx2g&JT4ztCRB4LtdQjfV`hj58;bxREPc00TQ<~pfyNxs_MWyrbFzSJ15 zRU{X^9dm=YuK7_i##eWHlPDBf9v#+ti?F3AGmLutLx5AnR78)lyqF{8$w z3B|9B7Zd1Guq@TBlfn2QOS%j5(i7izHht`dE>+lZd$fB89}NqEN2#-Mc8GL^+%~wcihXdjbK}^ z$UIT%YN|y`fHU=vnbd9Z0X^5g>W5%vsiH?g{w`Xzq8wwce?vJ7&U&&rn`PYpIZ{sn z)7RKk6s4G|?)ATqAS_9CiDa#gu+GPhAqSSxCW0K;6q3V%qFhF9Wy%9=9F;|2Y^69i zi89kD_#CsTy01ysbTQ_$EHdi2 zO3eNrB3u4kt&;qQ2;p-Mn0O7bjz`fv2J=eyAkRI24r1<;Apmv!wjlG%G|?5_>LJg-vF)#v-B86$0mR}tXKC6{#mhbA+IeG8oRdK6mh{?=|H=&q)- zZah>-<5PJEp@G5l_kvwcy#@kskQN2w4g-Nq zB4ZVymYE)!uD0_ks33A{c;ebav^m7$LHdT-e*w|pLbm%UY*{iTQ7`ffTZ<$9=P0cC z;$nr($aKiBMP?#E6~|q?Ujdu}WzkNrI@`wXs4PWqSC}@f+m$SqL=l%qePk<4GhA&B zBgqOA4Gg!Xguq@I(n!7+Mx|>BA7d7QFeig@je>mk5M*J{2l132$2E~6vlIPtRuf^VJFaDX zY_e#cljcjk;1PQZBPTB9wc}P4!d;<_<&4G@&aOcXjF)r`7q6Ju=gQtWeK*@s=S z#G0{0{$Vo9(VgWhmdqpJH-MGP`vRQ3?p(xwL{r;ER02IqIb6oo+A_ll(`bt6J=MoV zHCnCpC}_Z!26W*DGB1J$3L>>uEW=+_*ww?FBsY2=3CTFd)}RmzE|W(o6tVCr#y3p_ z?*pov1(zj`vu5CA$)l!G5@;D=!;HBV3#w_q!Pqy}tIR-03o{StV7@2*C!|p$ZB!&? zdCN5;$x3Q4O3UZT_vnEhb*oE-`9s~k!Ov36_B$Z6rVDN~TUI9tbRl64PbYILR(34H z-7g+^@z!^2^J_bTQfv4@AT5!VQz4dK0m2;o`G{7y}%N5sGx8Ftdu~4Gyh_tOU@QXE}3|2Z&M>ggP1v2adRxSsB25 zQR?_vq&tl6kRvZ{W3|TGs{mU%_I;xD;xz*Osa)ROCz4hsm!5_AUX(NH9@M&E?NxJt zXD+DE0|mGgG;y}44EQIk2IeF&dy0ESrcncJ&>vTaaDnuY9|t0u_-JM|-g9jIu$7|)uaUfWYLq$6|*~u1o1gaEQ3UrlN zAxJ`kEH_Od`0Yu{g6{;Y32^2WE3FM-R?(=GV=d86oBbEI)f6DE6zGQ0Q$pHh@ zhw4;aU0p3g%SF;ruKrUfM)bIeQ=+D@YmNewC3^lf3IHW1{Qj+g%t<6w*_~mmB9OwQ zSBNiP6T)ScdngiFer%D*ROE1MBuaoyA1B{`5Xj6k@ud zD6%+5;qtg37oiL0q?Ka(wZZb_z&1>0y9f_&uT#WCoUcvSBsTAQV6G-dLyVB^1+XGk zJL4P$Dfo0e6JT6hL_}LpPmc8goNovB@#L;F_7g@A?8O)C7v*;}=BT{ruh(u+BqHZ5 zqp{Y8MI!3MD6qIzE{tqDSwG@!5qbQ#ca6jcqG?#2)U!ibdn3r)6ZBYb+$6vaUqkcq z7m?z9g}HF{jUilOZN6bYmsN#10an8e!jbz|RPq|Gbi!+;80^ z$R(^`l@e{66bW?J2i>P^Bct^hi_tyBIk%0RHuK-?!}{WgOt6AXh~qNRkg4tuHYb{E&Fw37 zjNODdCk*=JFFWoLL`itrtR9b+e=xgsp!Aq^3qI!#pW&}lYcZ#aCb3SG>09kU*j1Mw zU_Ugttw6UnkG#c6BHgt(tYG&w>y7}F4L@u&>vU0$J%k5=w?(=D^6Z9^KmMKLr)%2r z!<~rWYV#tRr2qAWn7drYW!NxX{i?)dGLcRwe=RK64Zr*nS&k^7OC$L%hBVL>~SqY?|?o0LMp!l?msIdnXRd@Y2A?m+1@=9h#u<#xxm3i{6xR<_YtY7fVgCxQQbO4lu5veLR_gTLy0%gN`R?| zd;KTuOANO!1H*Q?6DnZjgp?XvDol}(j(I4dt%%%-*QA>K;XT<7=87O`N}@n4Odu)f zRoJH$!6RXSdHkxXNxiz-&O^8&UFIPy2&l!oH4y&oK~cS62Vs}Lljh^Rv;;;*CG{LC zpb*A&+9|R0JfUW|&Ud6&x?PZq!G(xet`m}6q$a#2hsRACAJ645uR+ISUBsd8>AAXf z>c#&Z%$6I7G~eXeKDnA)wA=B~#tlYAL~e7;vUIflmLP9LcFo1sT}KOcQfZK%P}1lcxvo~yXshre(N|=} zgu=4DivTw#`i0!Ft`_J<8j-GXD8@HHnC$Qy#ay~$i;`pBI7?KkYje_{UQ^j%05JDN zBr0w~3UHbT$vStxT8L}O`$@^ib%MyI*`uA$@`FS$&(n?{$!6@%exA6{gNp>ie4ePQzG>DV3_oRAU9WMKN84XQ=!vnqnrzhaOt_~ zEn^F9>bDb0f%HTY{onwRR2Q@LcqsoZb$L`q91lSoWc7xj{^z2SZpSB%VxR|7fbI|vQ3L81nF-C@OVxH6=nOJRBK!R8roU+ zRpjNjR{+VQe2bTVL%5g@+Up3&^2YL`gMiJ!+BD8BQbOYe=AtA|>#>s%7e(po!R*6Ps|fdWG?$f_d^B{ojrijK8K$~~Q*@E0jIZ24$iFEZCs)#Rl3vgqj_G&4}n(gGGKN9cuW9)aWkh&+P-;@ho-$a&52HhU7AJ( zJIa_*0=$38y#KlYGXhmKF6K}0-{U4th+}_C7>=?3Lq{*d>_|o~dXn$H5|c$hdb#Em zVQx+)6$vaICxYZ2*3Y^?nYTR&$hp!-Fkli*PRM3l{}Wg}8D%y>%Z3rEMm1uK;nW=O|D$*^)X|wm!p|1`kPY~+T;zCEleq8_z)TNA=^(=p7ZlFIhe19l_%T`lBrq$i)-m&mv}e=!CV9b6H(DS@~?fxHvjn;_@eV%7iI*UAXE0 z9fa%0n?R`Eur(57*63ibk9#CS@(^TfH{x6YIa$`mqad)iFP^qP8|G33Y`U^&Z{3xx9?0XWFC^!j}E;3y9~Q%GpL=JR=$wqN4AN zw?$BtG-t=67eY~0j)LR23g@26)ZE^c6sBhhbz*%ya2cQHKR6wavo7LWOYxmwb7v)J z#(kIY!OURVU;=ue1mq{P60ZnwQ=_?Mmrc=rd?^^^O;)SaqT8tW;7VnGZBU-(B%zib5J53U*(=NOj z4Z)cUFoC|*jK_tUcQ{dH;C?57gwp+k*{^}fC6nt|*!EjN=9HXQppvGqoGg$!_YU{O zSqkS|;BDGeob!hQ!+npnWp@k91E$iHHN_Iwf;j0mU1v{609#!L=mIwNh*6v*pw(Mp zU^)nm^Mz7^5$@W|bvPhn{(#tC-H>E@8lHr5qt zPB$fYY$@9HvqVOM`{y`mI-vP(6-}Y#mW?H-J(jxxz(pEvbVweaD8$K5??PSGCOVCj zVa{4RG9_7SB#sv4g6W11x)M}_vDF{_2XR)I5<7|VWTm#%Rjy=iv5Ro$tgDEWWnUq* zFj|ggoFpQ5G&JgIih8%i(l$|+X>B(W&&6fJVSW|u|L}QT(?za0yTmZtlWx8mj|UD9=iq5 z1(cCz{O`}uq?w)!Nu6-ZTaCakuHdGYA0nK_xJgN@S^KU6Nf&}f<3J%)WjgcDL*>`$SWRGM z)c8vgzcwx1AAd@S>!1@G#0p)8*zIn|E3A*=PeRmn5*y29It}HuG?Va#=Cd}1I%gbnt3Q=u&uQv`7Vdm2%(|U1e2{A>> zr(zr}h&*+2$z*VbNT;dc%3k_wA-nU*-{pQVcPYd<-p^ql8`lb?xXdx}^9KmG z+Ci@u|3d)%4aI2EcXvGq?Nl{E(=(cJt}yq>e$Kh#bJ3YgS%)Y|YKOl9J9EYi(}gq~ zlO6(eh69yRR-^p|kQnyrX(H}?7}^zUSn|!ON&HJ;PP{Fd z;;i-v0DQ&?c{6r=)L)BBP%jAJD;80l0zOv!%eIVl`BMRI-F~<`iBw4E3R4UW?W(0g+9S4Ki!qexf4W{gSZc!7?D5MGb} ziFU!e`qeAec&3QJk#RLX`8zbcv3*wj(&JvQi*}!3 zoU~j4#211+7UYz#*@6GA=Yi<1IK_(JDC}fWDlzmf7vL(h*6QXyPMBT*b7|nRw2Gey z$*I;j2SdF3Mi4dNCarqTUN4sEFf$}qEf)&&Up^60A3xMesnR#gePDT$i_h1RMvLLUb%qN;6wcHNHAIpDPELb=9e2s{ici0Qi z)MpD}4>?ke84Bic-tw~w@0NTW*d@Xah-b?iB|xKLyLjUp#PCpB0sFLl-h`yPaV;a6 zsNNvj=~8F7&k+H8@H++PR^x85n7hqe1td=zvSd5Bgz$-+Yy%$=fs z%bOxxSe^Xw%Jx41Tw}xw-D2HZNY-I!Gtjz-H)AFoqp(80TKMfEg(@d*v|5i43ee$q z1+N#GmmX=@>sZCtV)z}<+)PQkG!)YWx!|~ai-X>UbXjpuiuURS0WhwWM9+KBx$GR9 z(tZi?FX)>X@AoT8P9xn8-|wGJ1973fAN!8mM7tQsZ|U%U&i4z5i-;&Su_+PcX#=*S zW0xJw5B>$1I}7g>!pymZz-qJ)6>P;JTpPVoGC`y>vPJ`U(H|5EjH0$gi}#`svr12Q z;F6ESn%L*VBBi!Ic`+zv72pZocBrb8;7;}yvnBqQe}ofAlg!hOUnit zYt7G~nGZN=K|{emboDcY5l2ckjmw)vn^y=NsFk=KDaf^uDVDr3RAYgE10r3^pKgEL zC`t=EI;7VWZjFHnvHs_fP7q-}sn)83=ozF&e?rgyuQE=ulc$CME68&fx;#G_{;U6h zr96=`;to;dInv)B%YH%FnP4EAyeo|eGGDn5uJLw&kesB%a2-8-F3b~vN`sp;eTnaX zS->cZ(6(A{GP~fM4R3VFJ9w`k_wuO~u9!q~rZ{Lt*c``kf+*=!E&;CozanJ`bZnI5 z%sX}y?55&ELtFewbgM#k3v-7c7r&2_y>yW9xG-0ejk-K1%>6ZpxsL&2>?hL2V zP7#oMusqpaDB3A#g=fZI;2RK4S`ECMz9|BhOPm}u{}$J@h8 zeh=Cc15c_^n6zwPFVw7%)v{ib{iC2(k3+@Rj7NpIo)UTJ?W3{o_X`XyvP>eH@pEBj z8CD_e>9!L<={n?N;x!R@$(GeBM|_ld^6bSw*r1TeHYL#H5D7;jKOlz7!wY^gLYpqc zDd`MWyC&kDQpS^l3t{ITAuhKKR%x?M<^W3!!|-Y=5l*N%+}@57K%&e+Lj4;Zp*LBG z^v~(vaCSH;hD6YtIDBA1d`(p54jm(j1Ww zL=df(t~U|oh#waJjnvcY<4lpa2+XN-zksI}<^iBrVT&I#6sB;Tgd@1Um#NWy_A~Ke zrdYZ<+oC=%F<}LDnj9yIaChN=Z7|LjZLTxlNpt$S5SKPx8=@e6KLW~fhYUfO3N-m1 zBA6o7>(YuIQAX0{O|qfp`SS!j1CD968eX4&BPHgpRp;Y7rAz0ATZmvfpe>|y_C+OB?6i&&>MHB)dYm{>u4D6msjVMvB_dm#OM~c?2(2G7 z&0sty+J)BJzcN2~cM%{Ot*tKaT8l3RMRCSWSUaXEOmTSLwGpRgLWWrJx`@mwuA4Pt zfyDt_EG))&8cltAIf3d9I;4!JM7i?$$qkm~CBV!u96E| zZeT#i!qFLr33Rd8<>3@%D2@~4v^vn5JS@_k8l!TIaAVzNiht-W+}az6i9acz?%_5j zJ$!AOyH4>h0& zr^q>fe_SY9DXu;--pJn&3b`QG8|$uA#KnozKx`zMxcE_uLlriQ=`A?Ks>YuLX+mS$ zJ3Vd}We&3}>$vzOA$efyQ5R`Ez7(cCS7#hNh9Ifq?JaIqIQQ4&&-l?2;3^BJ59hB; z4CjxHU7E}95#*lN$!08H=$N<)DCMwYY4wgRg_s50+QR-#G#O808h%P)8UxL^USIi7 z85P5cwPNd4{h4{W%6qWqi*zxO3z*BU6hP8!&!nA?r$obg&gm=ho(OYO^wwX}{Knvr{Pb@r9(= z^rw(&?THiOTZNs^0OrmZVl2HTNN!W+DyD&$B#0lhKJ1~e`G!`W2V+W_2tq5~(XJjRQn z>PXp9SGYB-5M)%hsrh@e(jK2F?8LO$(LVC@N&a02iI3%QKNl*0zE+VG>%Kjn?I6s* z+fp3wT7n8aM%o0WPX37S++}Loo$4$#f#wGn^TxLqo6}O^T zW4PNUNb`eM+V!VEsxZ(nc_D$FHhMP#1ml{rC_kZVx&ctC_J)S%JEtm|(G zM-yTj5iTr_b2aIlC&cxUW4dNsFMt9do}goTB~uLO$?HVVdd2S&C|Q5s6GCj}6Evqc ztPe%j$X!~*hBkmSBjh}YOEXEi^)JD#iJ2pr^u${NU8|}Tjj&?IhQw2?^uFOoB3gZ% zqQ`GH0&rD&?FVY{jX`KttOH|nh0O%~Z^xM;9l1w-4x4eA5a)*^Q_%)?#?OH0CCDMk z+HI~){1aY7x{w%X*{g))aU^TPct?Q75teOO^A}&a7%(3F?WP5QqaKbPZxY~QV7;~$ z((3m&1EI{kg&8j@>>{<}*afv;Y`l2^F^jipX4_m49H&K^IENEuULs&%Xru|evIU?A zH2Pp{G-UA^Ckllnv<tE}^G1L{$Mr$43+=GlXVxl#-h zO#(zrVE(-y8I4m%Mqfgm51QHGUe3I3(ciFiYDaJUHHYz|7ISV3o+km>qbw%cb>PJx zj4W0Z;G$3t?ID{nW;-w!h4m`GWxdY!1*naHl_(ntG?UP#skdw`B&RprY){*-gn39< zW6t3%8o)k9ZB;`O>1f;|NIip<@p*+^3LGYN4Plqn7EA8{==z{<9*8@3gl=_C`VSWS z0JJduxf-AT0>U}y=9UhLSKk@P<&yzg^00ZJFh|2+k#W$7tIKGdU2uB}+4sg0V~gT* z*uMxPyEJc{w~GmOB3X53g}sCO*(eJPCOWX>E%>;_@x@}_4E zZ#K6IapAR0;zVgEKHMG5g@+^9%vWN$J%G$P4j0(dG5BMQKywbMWlHQSnvp?MsHfc` z(mib|D^k{4d@R(7!znsQXZ&JMAUCG;9+)=tYJYj4UB%{8Uyco z+%4KwLve^xop^PxB3a6u-qp~C4 z7o*b4^Q#Azky!BCtiJkh8HvYur^L^HRb+`_mmaa;&1_*gOH3WoRz6-4NMo_cjEh6| zfhZbFew=<+BFuZ(?Zylt<|ZwbHrwY3$_eXnDYnu)i&XyEUKwUFjXQ+6b@Z&KlsL=- z2kr|>0T?aSxLQOmKxIU}YwpRP@nXQ`-U^?+Ht03_!$tBdp2-JeY#W3JkuFSNaxK7- z^=j?!%kKym+JFi$Uokwa#Gk9s&YLsezRI*%yjG?_cFUGzFSe;Gh%_T`B+Pq|JW3^q z0wI&KsB}P>D_FM4DHQVE>Yae@c-gyOa`9XhY}POVO+|Q-ZQP?m^IQ#EaVOCQK>xH7 zcl|n)n{KV>7U6^ul68JZ&%-U2V0S$(QS&@99*U0X26JXK0(qsi{dl1+Zl#?IjcJZQ zXC_b*=YXz}{6U}@J8s;>n7;>-jZS-KbSP++Ab6*d{hA;OiB!Z>8odx^Ii5G$Vjj`V zl)YGJj8)hsRJx^GXAyP5w&>%76UAQvd&gSrEy(F`dt)>`gh+{f?Z zuQAddW}k4azB^wgmGT4SwsiH_ zLx4NC=3ka+=C&?@B%T}<$A~CruG@-CASVfNm6HbWEYU7L4;auDk?CgNzrY5t2IxWz zGe$A-%ikZXC_)}q)wC!{ge^hLa4=()B(drN#N;>K2aIb)P$hc?VCX-8paF!(-hMq( z^p*gR7sghjT5Wl4h40&>?upy&Z7jnwK3&_H8TW=-%{1zQT`$u0jKkxK=MLDVYXQ5 zFen#O>SAdGw=N+@JDxc3!6is7;7jH_doIA!2F{1$5|L&P?;%Ucu;LK~f)ZC7>1~9# z{KJjmyyu_oNI*9Rd2u@~(~LM>d5dL_D!!v-!@;R&=SvgEdLpT9j}#TdBHSfW3~~HV z$9S~F`q=5{0kC*pn0bay1^vu=M>`d^0E6*CCi?W=)WOG;QE7_)_~EeyAm?QD*jRvT z&Y3W)ee5NKBI24z4Kn$-A_7u|c3am7$@%xzdFtya0l5vZudP>cDRcr*9$tuF>>b+U zY+;UuKa}LZV};)WnWr|zkji7sNku%Vt8@~&lc3CIESKUN(eRBIni_G)$;HQJjXMy> ziDnjJXBRgr>o6bV0y@r1&zOdU_3axpyL*+A|!Jrxb`I7f&Jh{*+|Npi?TWxTG+ zuuhg|JBN5VGx}Rn7Of%71(G3P)Xs&5J4}x~ZE=z)%8p~d-grucGnR^x&Ns&l5I4n? zu`9*`B3)Se28yRj?0udSkn_AaS_ILVJtS9NBvSRn$H4US9oE&?+Z3hgk3RJG*w4QRf$hTm~+WvGYeGM`amnOEn4BR7(gP)-x6eO5f2&CRM z`UVb8E{AflFwsiZ!W)-RbAT~GNsqsxh=rj>x_q&~l|Zl(NiNn^7+xUlS7M+1p;76M zHicb){P-`j-RZ4Axi$9MRYY(P=|%GY{?$&Dw=Cj9h25Chr>sWL3qms+c}QN+9=`{2 z%M7s^#*w0(X7XTh4WznDx^4eIQEq2BG{FMwTOm%7xe!O~opH$@3P|?nXnf*BK~%R3 z_2KWYB}{cWU8}~wM0oz;Y>hSkqU#*92QBJL3KO#jktIGC;jFPgl6lACHx&OOKH=JW zj}l;+(WPDi>SY`U#3drkO)38^KOgW%K(i@%-scV_x{OPNng_aQ@rFqEPpM;Y`_qU; zZv^vfmdq${h=5sgOcw0=G#eN+#q$DkvHMwMSU~;+;&Nakz`Hedy2yaQ%tmsM4e5v= z|Jgmv1u5Q0EfG`_Tmg((ZX!-=Sh419B-&{Yqf4sGA5|P8FgH3)j3@RuQxFx@9L&hS zxlBgJtnr>`_{Iv=6OCIST%|Vp40rYZ3}ANf7=Z20-Vl}xiyv&Jqg8GNF@usRlsrIF zhaSQ7gkfGFJWOGie%$1>C&clh&31(QxF|WpG7**+;0^hc#CXeJJ(6SzyunZESp-SgDs- zawb7mcM4O#Mlg)KGr_&{Mtmf~9PDDz?TiU`7Qocx53eGi;KbPY-Awcp-V^>xv^mqm zQ`Fns1({2aB3o7+NSp@?WvH^wq1ZS@l#31P2h-z7SLPT@s+?t>em60kDKqX+JTDqv zwK0`^t*|*EqXw^ybKC=Hmb1L~HQ1z27Q{a&DRD1OnEcaLF|I5lx7@6}DhLj7vOYcj zEy69zS)?Y@!F&BbE}rOA|Y*K7U|7WrqA%6j>r>#s3t}e48?E?U;N&v^lMH zXq~u8fEhQ6{lhIHDFJGl{&+})tAon1jzi}7$pb(x6h>JxvY9M|KM>k`V)sn&z;Gq@ z%S4hscYUx(tF;L~N01u{A9|dvqz(HeLS5qN>Rp$WKUUg!kpn(FTK>oxW)kcflk@(S zAE^mu)UwQ)Br>-JV_Mz+4}#p{$+@n)UGDOb|Kb9qgn}1Ex_-!vb@WZMJq$wqa6B{^ z^NYx1iux*n(*&8LddQ$R?i0{rk!09d<&h!^X5}19tRcWrm|!F#jtFTLExWt8SD2dy z54`b3`4esd^ziOMj}k!*joM|i$BHCz;HyKhy@j}7a^OE2_Y0u=VW`GXzVGqk2P_ce zwtB-S3WPyfr6(OwA}BnyC%#088Gt!40%0}2|F;6d2Mvd9JW?;nC7;M^Ty>FdXl@>G z_>o=;I9aG0y3WJCaYO#8y^9?XZ^%AbBr9Ex^y_B|aMZ8ZQggfI|`S zfx^xL`!DoMPeFUoO7h3V3fXV2rvYhTj+h4HYY}9TCT;FWJyRsWG1)*2izaS*f9CE? z@ZNPLz7gRDd!N zhh*_fAdDuYagan_5k+Ybn7DxZLJ?h;cUd{BvEYkf?gHGS(EKfc|CwV+?Fdo)ujiLy z_Lm^cO1A!N{(d?F|TbyFs>#bBpv4 z;O$ft57k&kP@Xkpi$2YWw`aJzmBuwYyIxaKT|UkQY4$S&kQD+V{RMcS*4 z1iAXGxYJ_G{}%<5TkISKnXl}?v07ng5)TNZ-aXvxKKm_)Yu)`=;1hQ78~6;VA^&_L zhzg>E6;Hk_V=z828mY#2C6v|yCgJ8koMt!IbIw)Rm10&vgncc4tYXyut-@3gP2o`N z_;wMqo1=Q3BYp=!Jz7t##08>Ull1sf>LK?Dbf@T{$URzh{w0h=la2Y3??SjrQX9(y z^ooKq*SKvQx0O-)aTcoaw&;6AfkC4P(wpb7u3a3Wuv25)CAn#Ze-@CR?Q2G#APRsF zww}0NgxkL=uNqh{zYpU4xr^5s3;e5~QM;}cJC!s!)Z$ptPC*xNXq2age3=2)X?lipCB_4vp*C$@`KVZupC+Nuo+`N1al#~hOw2qMRe}Q zS`)ajAjE&A?Tord!kj!7OWDwM*N=*%*j%>96G(Urp zI$t45Oyzhv#(YvF(d-&#ldL`M8UkHJOuCR#VnZPwopNY*oNu6?9WJ-6i;!z}#9quc?!rk4vbEn+BaJuKgcknd?;A>XiG1x%DL|F;Zn<`hZZU zhg8+4_e$FS3j_m(IQTkJVRK70fqyrD0ip$9P@2wLMfrD`yQy{W5|r~ss>M97H|`ad zdmg39PGTP|f5ufOqQGE0DaiRHHD5DU`4WT{WxDT(HNJ)*^Yq4493{$`bL^7bU%gR6 zotzMHz8BvK%EJJY12%%+0=R%=iOQX$kc2rY-Fed;`iB+GkZppSvFt3HB+hj8^+mVB znm_8uxnGKKv9(8~TOK;AqvAC?{-AJ9iA7I)+m}j6!mOmNR|Gi?1W+Nb2_Xx;l@wEE zGrF%$*do8QnhsGV?g4#17zg0qPt&Why^%c#oRrp8}i!dtN4f9`qGNNwC3ZAv|#o zC~73dX`Ca%D<=v`8DUV`X51y%b(4dk)U1={1oIF@eHD9(bg6XnvOkV50Ty*G5nWaS zbV@qb*tz({++||Y^DMUsX)WBzwlqzLPYO0?5Gpb8)+F;*85aw(>{P88U(8KhDmT;&kae7p-b-=5tpbe&t>z$jQ+Gxcqp$vGL!gJv{){?^05m*xZ}u8 zI8z~ljW9EvyZ${fU9@Y$DkJgXN+D*k&nhKS?z>=-l+J-U^&O2qVJ;^p3@;4jOsh5BG}Z81F96{dr>L>eB95*$=4C)p zd>QT;jwKX!F|;k#JuYf-qEOn9`y9P2LdgD$0=R*6(;J7ev6P_P@^S}Mjg5pbtfuy3 zqE(HD%YXEsS)CKIMHefQ!E&vuie<)8<(x@n`58Swl8$_5dwO($i^A^=IAWlZ!`1TcPmb1lY22ZD= zb)l}6jwj-W%R;MHqnxQkMU)d|<1~o3j8lZTg81L6sm16236RUr{Kw{aaRIqsF!5=r ztVO#(mu7%F7{`cIHzChs6?QK2DMNoeS%`DNmI>zxqcMK@0zv@dN?uo-D#-cNxN1cI zW?nf%APJ9PSF!F2gkg{#o{MutxTffdQGDl*D7`S%Tak}uj?5#Ffiy6?S1M3MC6?Kw zl253MD_?gM_k6`8ALO&4SZ@r32e5A5q2lDFY+)`beMc;zqaG!Yzp#UVA0k|1^pV)7 z;*dv>3(Pwf?J<63X!BXJWi7Uv3E*k7uILuvlA&H0;av1yA+8rznQXw)yRYx80!q$! z>SkVDwfKomi*C}-vl@tN!2FF41Z-Jh{8DGD6k|oWuvLUSsV)yF|3RGN?Qo83t`J6( zv)F61e^VJ{c!09V*Ts`%N-SP_uOprkL{9jR=FyAQA!yACHxAcRnEGQk&cw+)y7W(- zzTNuA#v+L;Rm%dj8MKI24FGoOcOulO;pW#l}KAZxk1DQbK3AOJIw8z6iGPI3O2cxRDRe zt1&DzHz&86I+}5F8P5(T($IQWXg1u@fpzOinR&(+|5xit7rulL=QB|6?Ueg47RD8Y z!g0NU6|0GG6_85WV!iy4Ii@Q%$pl*Sdh9WQn63wlgUs@xSCD%SJN;U6eXznrFf$UZ z-{jR}iU8Va7oBz_-~-X#y!pz;K>r-5b?A~ z`)kwd8gdkq*63!utuHPQa{`wLW7D-koMp13VWt}rGs#s z?@{_sfvyOhQWE~yjX+!xiqWnu;O>Il_?&F9&@HtwK$41i2{?8ZMMq4#fqg~zcl1oM zSdQa!Y$Tm(Uu?7q5pu6)wu-Gvx>GaR<)U3o-7WVH?@57V!Et9j-V@OpG!w?I9^VR} zo3*1?q zcor2-iP(=LD~oW6WI4M|%)2=NCF(=Xx1z$i8G46f?@ZdGVeBW8bhtT%x@ijtPeIx( zaTY8=@4FJ^&RPRU8k%2YOXB91Lxjh4z9Ga#^#y=Bul^x--7K8K;1NGw%3+39oUv7r z+tfNYO*P!u;R$uRRjf!^-ZtGD#I@rkJ=U|X3`jvF8;UqofHj#&2+ZZ&I!S}~4Sg_w zIhc|E%&wSqn<4||B`ui?3ULOS%&PHh83l&p#3()%qM3v<=yBT?3CQ&uBW*~C6X0e7 zv(pCxB&m(k(c_lwpwvOJ+>J*>@QL*t-mJcT@h2}9Qk97UTw)EPB(_(C@jp9jJWuQZ zVg4kSHqBU3h`T(yHJpD|VlP3AN*vU_R(`ccyb}BG$XDmVhQxWCEUdNA;#n5{As8;Q z(@aTo(TP4%7v#DQrAw)&{{n#0>FPrKSA;wKAX9M6w{sDj69g;(Y62*}9fe3hnOX<2;3(1~MW$gLegZ zn~1lx$WMZ5VC zj?v4;J3{h=$yP^doo)62a@qR&(V9h<0P|I5h+0}=KVhEeFr~m0L1y633Z$s`OsL0m zBB+sv@vVEB;#>(m%RGCT>c<`(mANRVPo=Eg>JaAilLz1AmF#ALG=wheFiu_&N$oH( z!J|SgcGw%*wZrw8u0tLpq*Y2?>yvbTia<))Sz+=!Qv{9Dfo}cvl8LB7=?%gsP zk3VSB#9?3;(OfAWOw)~az!e0#22-Wkyg{_nqR>qg!0|#ID0K)#s`a`d%uduj$pZta z>>)IFEoz3Q=4v5Mx7mxe>pnfuPM3!<(5Z9bUXTYR=|~aEf;MW{X<@^x5qDT`krors zxOhdh8H>YIEcO=b1E7O(*MUiVb5Uf?twTJ6U`Z^xm0%jedo@7lLZkSqJzZgclYO2XyEu|!Kt8> z4|PX->>$GIZL6TPJ6tpkkezkI$=igubkcRmGUkq)Lf24VZ~S?L@AR-XvcFwnHy`tY z2Hp3jfw%~8lR=y=CT<~gra2v5Aoj2VT@-xU48^^ob4xH}G(7Gb1@buJNDQ9};&GLG z5CuHY9Hq&b=P1qog7d7DOv*1V0qn->J@Nhhi!AhZrLLN6ASgE&*K&|X*ntT1>@S^0 zqm#v4J?X6ScmC}|qaF1Fi{B;o!{VVhO_(c^-aBN1y+DxKfL&>PBqCS5A2rxlWg6M_ zc{Aq!HF0QImIAgaD~ocEtW+w|SCRw$wK!EIiPMvKn(8-2;v8SIj)*29x)7u5KcZYT zY;!c^;s*x-xeXcVHB@Bt3ZaE%WVorqc~Tsu3vy!-M*ODy2U%067&C;JjnWWcZ^A2( z!pICS*=ZZek536=8X3E0{OMqVd02V zFBU!wB6laQitBi;W$jYUXo2b3*KRI5hl8?p|Z| zBcaG zRjnQI{gc3)f3pi2g@e`-VE}PTA1j^ggrwbg#l<_#39OhK-O(C9V>TH(|I}=i4gM%78RfuvTQ@Op8 zukx&PR`JUKGRJ-*&F2o5YutvOozq|j?v3q4QRXfVFRoOWsL8CFTT(IRcK~jIbl|Ej z^^?NP9j@RrJv8HMLDW^xuf#Fu6#oo2BuVnmKTFIz2(UNLEz-eQpH{&E_>2N#-cCn7 z&k7=G)~$N1ejbFYg4&89w1EH$&&zBvsxUE;#p`j22(zsR;r8nM4aZlNxUT%hL!j}3 zNUI#Mbi}FEfb^NH5ViC9kSi~RuMAsm7iNy@QLrJd?ot02h35Vx6P={WIP*e47nTDj zwnR4waIBc3(Y4gPqe;GNBmO-x_2@t z+TxlrQuZH*^zyPmc-5%k5aklWZbS7xrl0sgkkdfR+Yw10o1%^ zLeyOX>XkxH}x8&z;gm4vk%qE_^ z36f!@>m#*T>So83uhUpdgxRfUTWw@8B+zW_t8}oZ8WCVFGC6UBn8zl|A7w*}luGA6 zQJ`yv;)Ko>X9{u6IO-hiXX=kjgt>IcJ@I8p%HH?JTlh+05#(rw*>8n%QJOU@NMlX` z{uw`gafwLhs3D0A@PrW8hd1T=?Wx z9+5S^yA6!|HG?6oo^l7I^XKSR9p^zI?oWDd2T?uU+L>}EFxBGPHsZp$qWl~3fSmR$ zcNd83HNbt#jYZP7XwN#MQ-sUWkJqIEos+cv#SyqC-V^(aaGiBN;>P%mz}y(gT~wSY z#Oa_=^Va%BVQyOa|CTO)vAcm<0%bcF4y@x|MX=e0*VJ0pddjotp%_$}G@j;h3I+z9azFOEHEW-UCoA(NUIF_QSR1gTOg|PE56kRRuYJ zgsn&;IB@F-f1Cm&}GZaGtT@ciM+EH^C0*GPPkRk2Jvk)#h$2+LT z9~O|ShZz_%Y&GV3&VO(`R*79in70kRZ`Gz}lj1O;E^pUxJ#H51Zi972CGN^8px>>< z^CB{H*!k%sY~|;HoGaVEv=@)9g=u<_3FjV};07f2qas{rylrD@wZ#hsz_G`K)niuy zt+Q)&0pu7C;mB#;7x_X#En~-Ykz~SwStBOD1mRNf6c^1+-*~pfSk!z%__;6|svVP} z#a|}u!AbYk0=JY9O6S*jndgoa;GTrmXh`N5UAo+K-YbsC=#wA2W;`gsX>r;)84pB@ zz6z3upQg9{h+G_uTPem#NqlZ!0WM6tcZZ z&)aS%P+h2w>C?_4oHA+_8vi5#&YD$kDCU3LvC)U3hZEu0xVNL&Zwa8-EE^R2r|&@J zfijGuljPPEln0hxwBAItIn8$|&@L-u;1F&g{vetplV|gIx%`U54K}~;@|6;#ZK#xO z8wkifjN3Hyp;~M&%xxp3gYKP@(XK*Wl>Q1AZ;uk~(&`4ArkWE3xqb+0h}zc+aAzDq zt2X^TNCpypJ(m7w@k#e6Hx&t2`*5QZ11&<2k?t=dH>Qq2cYYt*nY5+Jw<>^Cr*n(+ zG=*Iq^=|Bo^@Iahs}TM1evYoE8K(S;PfnYgBIDvD(G~=xrDAKC)|$rzo9mc*YMS^) zh3>{UV-tg*Wp zia!gYVtPeGO1{5|hA$k(YNQQ+4#^(w(Lqoqc>Qn#7pw(Wi8NQG7+<>yL9plemm4&s7)(sBgs|GokCHanP51am^8& zSS%L!3cy{i4o{gwR~16G7~-m2v88A=0Rm9B!fpbDc={g?yT@Sufns+twBu0GxfkR5 zfIY%vLMV^o$EN=)K652X?+INh#D&r&$!1LX8UQYGrl3LaOOgKG#l;G4ObT#cVQT82 zlrlWKzC_z_k+IjTbBGLf+0oh^QDkuy0Y&NB_Q>o_p4mkDT@$oZ-$vGwQ7ZI)%YZYq2t!DXo zD`O7Jumh(DRT@Sqi19G$w*MpkN9hK^_#av}PO2~w=K8SBhbMCIwlHZ}{}2yycLI6h5^sojLZ6b^G3 zO(NX=u#!nH9-l49jfO=p@5qX#OHIVyck6i_9%RwIT4BfIqG?}zEZURw`0-<7;*TKB zTExKd>%?>c{*JGWSYSR#*Ne6j&#s#vgp80<;sJ%7I49rOU@TD3J-TXqh3GsrNb7)$ zkB+!_LBQO2T-@a*ZfCqIEO#-kG$CBBwGfC4Hh|fFC4MWwQP@xMT2L&%a1o`epJd{9 z0&-nzG|^k4%{FFud0`KnUbmnQgl7an!Md8P;_ zi2o;Yct?PnV+6Ci_eE;$!1x`D#WHr=C3Y*;BH$VFG%vs5_)D~hggk0%K+wIf6KZ~7$fw8Ye!istlW3vEe>c&bWw!Oi2@1Lt zeZ$u0V%Q}RW*}F`7Ye&wFv`&hX?$D$!}#szIs>cD8%q(-m2York3E-$bVKR%KAkG| zl$ajXO-sYEKPFyU6M5())?NlmO@gIRJR>5{HYVlfNCtaJcK#M>?j#_h7gLyWgA{ee?%0K z+34|D?FtZ1xef334~V2qP+TW}`dh6C?VNPOmdms=0qMTO*#caz^tv?6+jk|W$>fOQ zAod+oq`}QXn)E>d{-qn2%qZUDD?whZco|k+{dDIf=3Ke_m4Vk2V+7Dgk|6JMK25Je zsSzH#D)EO*pp55;UxYWMx*pFgQs-huXZ%OFi-1;?WxpCHtl~tb>x^yoRUtf{u?LzG z3yUTPSWu0L{o&j$D!mLOwK~rD^tiGxHa21XNrjuLdvvwNSMcg%!`;mnd|3Y9Z>T$@0V|` za2}b%gB+cY`6)nV2(NyK`R599BKRCaW3P?VVru~AiH>8w7$@2ZVL*lze*(-LOi^T) zGf7ceZOinNPwx+yDJ8ajaf)b{Af3lhf;-E^dCL^-(lavwh!|MEuUQm_wXQ#&7oCfP zLZuO#k2Szhw_W_);{fRJ01ckAK0%}_DI;d;O$R++V9}iT1YK=>0q{gH)7?V?;AM5X z?rIPw#X2tvvvYq)Bwd9Qm3llT!j+OGryNJGGZBcv$bBKY?kl3qE}l@(sp;~QK-}_N z>+MYjG{*>YG8h6eF(QizazoTHs)!dvyUfhc)O>--0Ir$rVYD$Fjm=9CJ8;><(Lh@Y zbdJa->=j5pVQw@OFiB~yW1IxLxU%-)=QH&)wU>bohoZ|&fu#0&|2Q@g(X#9A8^|*< zwVtLAu1_-4U__oS2#)p-48($KJCPxEpG8Gz;i;>it|ThgiWZe)WVR6kogCdXox`3c zC>PX{)O%$lb{0+9Vl2N7kzDakc8STfx# zh+=c5SVxGbRUT@IIoF4DskKw2>#Csz%9Ib!Itsc1Sh}$XL>nx~yeqBoC3E-cLzW!t zJn*jjQtdG!6!z)<-?$A3=PKacy&3xopiy+CE|%O70)DbTVsCc4D0O7~`+cpjYwBJW zV>SYDxoz-*$QcI+BsEURx?;k{#Xq=d#vUQ&`I$eXPHM)gnZSIpKh_iB^as)|CC)4n zG|JiwZ?*|>a_`rqqtow`g}D@{lr^8C`9D=4)$l_&XOvNpx9d#|l1JlQfo35VsGzr|zxKALtq?-n9JDK3pHTHcXTxIrX-AKU;|H7Y{PzU0Wogthn z$4+=V;yS({FX!@%u8l|Ptqbl7?&*|XhCJw(a1-o8((!W7PGFvnTe|o$`oA^kIkA-J zT8(Xmny>io8z0Y$cK6{f6Y8@=b_1Yy`R2@}qTnDJG08x4?hfhV*Ssx=g#_hn{F0d> zj`?K~F3p+<3BcYeWu=^Eudmyyc1`ydE7X&BM$!5G)Mv&+$Rlscf62Z)s zlUIiJCPJJFW~l5uxqwwEAY5)}$%-yP{tqcY4~YJ!jMRa{P}$fFwGqiX$K>*aU0U8b zp06*?guSC)!i=i~(S`_kwfHI%bvZt&wEK%r>*5j-&X`7%W&N_%A}!|oq^JC04G6Au z;eT}ng>zQ*X=&%Ro*)g_E*7oNGQonv%Gb#k_4lN@-$)ekD(ufLG}AmS0%7RTzcppoib7Fe$T3>WS;>CV1`>SK!?cZu-JR zFrE zboV31HUVIIFIy=ET}CY9vHe4NFDNrZ%WN|?8Y&R2JxoQj4HpPIIIO^8DIu*)@V68j z37{T09*%7lrjz39knP(6nZhW5hB!@Bu}+{7SCX5hirdORxH z&ClvUGtdMZO#>q%wA3gdwiDHw2=PT42MX{n8j8u&=F2697>el$)5kue#4}gg`bXR4 z5bKE&q6a1a(N79;fpif#)|?JajioKxS>fD7)6x>6-hPfySBlF*O{A(N_Xlx_Mb~1U-NEq2D`R zWbVS+;4>rAwVKiVHMp}z`OA6x#R5oT7(X+=A?PY)`wdlTrGr3Qqd%QjurvF23E~&7 zHmDP(IO$+wnyKtNlC$2^OK7SdXkeO75KKg`6Cdld>%%PA5nqXRB?iztFLFrnSw10R zo09IUjP}Q|qCJ_XEr{MJu{F~>`r>LL_cf-)!G{(p<4ig(7n!>n7fa)t!~8iJh~mQM za7cPtC%aqrf1)Vyxbc%>_Z%KV1pMt0;EciaD&|cha%YvR&KjEeCyoR&Lu4SMo$_f% z0eLXs2UfR{Il5}@BIDKV~f8%F`>U=!oNeX1_j=d`eWDQ zpz=zMkja~AaJ?#!vT~+`1Iy!!7z0ub)?`HYjX+q^%ZV`8=wqQ1pmSH|wmnMm#f6ZK z%d)d5SC||6hna7!K#AoO@e?0$g(|#yh zFf9J*>83h+M+ds%GEq5m?zpPuJ{H7al;ty582>BMS#u*)``XXW0C8p0^Ca?PHttL? z`k7w3k4a~l&@oqBE+Y4*;pxLY^|<|PkYZ-hlb0m=```H=REg8$I}v$4sCC9>=a}q5 zp~4z9Akr!Fz^F9#8=qUmAH=oJwj#}9e3fw!vA+OHM{gdC7e%-^(evOaIDbr{_>wb< zU)$=;zp;q`7dA<*$z1+;fiA&xj6vD$E_+__AC^KmUAkL<89M}{qr>(7 zRG^tDU2&hRVHdan%++u3gvVr&&JF2FGj6vKCr(RqIfXY-F9f2IQN=MJe|Zs>6?S#l;`Ije|m|a!CR3P8jw+*9kD+no<>fDVpCg=&r_Y zmzuyFczWz3!UdL(HN=*yg?Kg?ZgBR2H7Oc_wJs~-NvSe08b1@_ipdYGZoV}xFCyt& z71N(Hg-}%=GF>CeRZaaf{wu&$)iGmjly&R)E5MvC(+8J+c%|XWBATBhqZVrmQ`grf zyB3>?^az$IG1Evo8QEK~6XOCw>R>ktvU(@`k&NUR>o2 z`AcgWN-km8V{+p~KeXhS#AzL=A9qcG%GzW|0x<4`gt`)3F4YixMu-clr$rO}YmZHc#}v zGuu*m_j*>Sr&VNiHdVacAzmfi#YPFz$t4X2UIHoh2QYn{zm*Vp*6bfO>V+I8hjhJ48AK zPM&nt@?#-nhn@k81QC+J)YJduFvdV}>P>`Q%;AA5t6($E66Bn!H65-$CcqqQvJakc zvq?M&sp$_E>3)nY?ofQ5KeJv>V8B>K) z8>xcgz)WC{+Z$u=;7jhaIF{j##nwWohjg56cltm4DOO@Ygp(#qs(ab;29@_k*<^q2KbV;mqiKT=%wa$JXT;nm*?FE?~ z3^_Xq)R?%k{8`56bd8x11S9Su#*2#MTqV8tVkyAZ=x#uFI2IRxsRA5}8MTQ*p3}+` z1iB=g(dZgT%(w^0P25#s?Yu!WESBTxSmfR!5lr)SD}X=h0$p};W=7Al_^@y%E=Qu8 zX>S(fUf;o`=*P<#5-G*@uSoRe{%Hj`VOiHn7ryQNK#Y8OOT?p( zs5}hdA{vGj9sqG0YtCWH)l9UZP`4pYx<`@|h2u+%O@Oh>g=x@8nYG%3#ZUNO$F>`z z|#eb7Mds^)Mit&^0H-I|{m0^l2E^LFq8)bK$vTaAB_&-wJ3g z%IuDF>&^BEFh#(8Ywh@n2-?j`l-O9L(~v-o;hg}+7T#y#r;ieL!FYSB$IW_t3DRWf zFJ;<1^qBwZ=-{H1!f+5TD|{P`D<6j@d%AO5ypsvN9~d8rFmD;FY?y{(g(rYq9(KbR z$nGe>%pPtajc^)8@2d%Qx6o}_M6jbu1o=SorK$}gtXE| z+uVsY80xtvLEV*5;|-*9(!+%LU-~f@ui{%Fq|Vb7>9FJBr-~RDk}93mo(6I4da#{j zG{po#t{tyrA@84A#=%1@?ue@~_8H>9t`VKP?Ipr#;2o7V#!>g*fyfvSa}=)@&xt0! ztXg8mvk)%3TZ}<{)^lLF-t1$@;sYW4(xE}ncpk!?M*3(pT`Y6VCxUaoXNgpW?h#~m zHSJ|Z=;!7C$uTFRLDSvWg3bLdxgK2lMQDaKWkB*)?4#A`dDwyG-EBfGfWY=o;uiF>eA_ z5z2p5vL4%sV0`2YG7sc$JYL(3C0^m1d6LfBffjoe$SsPYFVes+0!WN=6@0CU$h^fc zvomQ;aa<(Wb!D1qpeNvvT}<2kH9oYM#(~!-qL~DGkd0q2J`HMAT`1ZaORL(9Zv>Dr z%8N>jdBd?W%H~3Rd-RDmzbkltWh%N=h*`nls7=Q^I~tGO-vo7mrNWj)itI1;7wq3h zv_Rs;@huRhiSd)xihXlfK6LvOcCt8p)HJLsMYsI})P=w0p_G+a<+xB;M+iM zFq~IQ*+?z#5J;tDnG(;4aOX}-oI27f?-UqPxgW5u3d@70GJrQJ&Nu|+nGEqpXG_(% zRG_)dOw}9nz667nfuq5A zOktM-p?#|6QJgY-0B8mz6&}nTBgl&uq6}WYn384P`I2z)fk$Oywhtl5m>!Ii%b^$% z?cSrq5ScRb%M~AiI!g%&DA$=!migGfQpPr7#do49BUkEp$bqnBlg|E1TrZMOpri48qi&U4C4?*y z`4A)GYLV_CNblTC`8PC0=;ZKYhQh6HteqQt4sGUf^wu2Klr6}aCWDOhM&)?|Nuy_U zIG+6vVK-`5a#{140BXUpRd0;@0)n}g$()g16ID#`!#3KiMqM!3a(Ab~$&@srpMD9H zy9!-{%a~sQkfm(*UQ*Z}HHY`aTLR1kM!syCzAO=0e=$;qzb20B)`LnEcQUcvH^pxq zTz{ndT~H!y(Xsed+{Wz*Im(mPX3 zf4p8fYk?tR>dx9|d{$yfQKnYiYc}Gwx~#6p3?e5mrQaspxx*Ez@TdUS9QA-?^i{tH zl6p0by^e_7JCsZ{ZW7}9BtwSVMZ0}{Rv=#8C(zmSrHg06!8eAcZ@|^& z1#~Vn2NNLJX2M(+c^5^9I8jJz)#YkjazDn*^dn%mhK{?G`>lc;Go2=6J6z zmtk?E2&d1UUXO{8{%i{Y=8if%eY)i4$$~uT^QvL&DAI#j#bTq((K>}+eqnG*hDBX$ zps-nj6uQH%DL66~KX(#I zZc}g?bcMn$GZ(ker&Qu*K^~rM+&=5_^8JcX*BFIU9s|E$43LD=MO=>K7KfxD5_)0- z5vmg#sIL5h<(6j`ML2z&Z_}E|+B61BX8EHmVRfl$yE5{CRF_BP$XHo%^*$3MT#2_uz%1knZM!yE2GR`0 zCkYq5Wm&YJK;rnGZ%h;El3-*tF}}BK@edbsb^d^H_x%Fh;tX$mq9C&*%pJ|{H;|%a zu>IFE0gMu}#o>zrsUJ26z439G8b^1`4s@6w{e%d4w9$l|T^=jMEamAb_O&wsJW_#? zzzO9)(y$|q{8@;&&N98pa)e!ZHX2e#(@pmjsD?%K$FHR*btZOd@v4YC1)vQ>EQ+~T z0D?CI?DpYZEFju!z`2JGhnlgVFc%!(IdTN_i~JY;s-Y_{b^Baj(0`4lOW_{M)0|SJl6Vgsm%yk|EEP_^zbQhzUVwEh7onrS>uB_ z5;7r9yN2Q^(YdK5!E5RKOqkPVI>r!(ZSWdJH15CZl1nX?6Gma>>||77YQ?L}U2%^H z_{si(J?{TRd3j*%veJkh>vw`_16|8y9$9xyNax8;f>j7XN05uAck3CVow37MFxM$5 z7~C`^9Z_A+|c7wPRu6lo%6r zatjm>3M17nrpzZ5cDs-&z>M(#?pk=sjekuj(&R!eeUy&9<3uobnJVi(0_Cnka$#^z zE!%(x%SaV8Jsh8`Gl@vBxM!F@W{7Yan8nd>alQ~r)yAQ5buGf~Ryd3qh`(leBhD)!4z$?|55;{Yq{n4-fIW6fkso)H24Z*7WR#pN3hIHlu%xBLj1|`*x+~KLL$GL=D8$W-(PeKxhncl_`v0gp>o{Ah>i2;%1D z0S6R@Se`xsNAPI*p%Xf(juHCH!70h+?%9Kv6L`R6FvvC ztL1Sap=M`0o=~(yUa<_A7g-5cHxx=K24$|RZNq zu^5&++$ZwC1GTaE?pFLu70~dl#9<;F6g79rSiTnGYizE~v@5iT5{s{``6riywgaU^TpUb}M?{!0SqhU( z>?Usyoa-ccnXC>#%;Vt-VzlVr0l?*B`!y8*E9tgr)27CWJI*A5|j7h6Mk^%q*bx8HzWZ9Mv>?x+ofyl`66xYm9c)=HIRGGc3%#!MuSwiijsrmz)1=_ml6d!Vrf z1*o$i-s_1CszsVo;$b5OM5X-i`Xhs6YqwTi`@qaXOd_jh76mozEx{ zj-hJ9-$FsweBO8=x_w-KiaQh8PY+-1h>h<8PJe01X;eV!&HsSaU^!!a3N z^I(!Yk@lr65iS$X-MG7IVTa%tIvG-HdgAhZAl{V7*;`Mm3-Lc}JXttBn2f3oSMGnRTq8*#@U^lvt?VJXz?RqA{zB?h!}88nPY}MVQ?Txg^#7 zg&=3yZJX4sXE+9L=nlrtBHWj(?;%y6-Z&zM;fU84JB$*BCfDPX6nTG91pNaGhwF%+_Yz7&rvN8T&mO z*2r1H9D#wt!x*B+cfM%h;MzkAVk;&dS_GV~dj46otIqWSVz(JF|6xGmb36lz$Q1If z5|uYY>CQ1H&3iausfaz3o+OH5R%9k{nF!a4DNSx;7CfRr@aCgC(nW;0*l3FI#JXtD zV)PiEKb5F-zh3~lJTixt0YAIFgqjPSTxd0xgL4(q4a-(vn*sq&zg=#Z);zMHIX%|m zfRY}qU=z|I+T7qBs-)$4rXbQxud1l+&J&<2C+AkX99_gf)|qVTjun)f6P<5Xw&R4j z&!QE=t^wcYbh{e`yL7DX)wo|YDa)xvBaS`>$`$l;^RVKUnP9FU4%C^X7d#dKj`KEO zGZqu!-bUfk4#`@d>j^hknHN}|b;|wLaeywRcA5;6-AkldOFOSMj(6$|l1dydBJ-xZ z#uH5gvG556l-x6s?3f@5G*U-jWcw2#@}fiwStI8Ffz%rJfLcW2IMFT$hr2P-IJu0L z9>;`Ncw!Z3cF1)J@Bg6AT1Lf}m6rXdgpfEdTS`wLKFKsQGHX#^TzV4!Q@ms$|NU7I zu2HA-QJC-E6J!R%KBkgYPA&j$Vps&vD*?8Un9={>mxOR7Q7DwMDV?MxkkYaP1LKE@ zZ>v+l$QwVhOgWk0xKxR6{)&HcLFn`HSh0d2`hui67$}wjIO*d`uNij;A~n{y(fCkA zG1KCEkX|<$^PCFml5uIp<0V?46X+VEx{a)AS%q@ux)@dIH!qLY#%|j^x4L6=-Hi^N!u?Iyu<`P zF!=$IMRuG`w!Wx{!K;@N^xrLn)w!*0TAcn{$7Vl(!bMyl(mhFcYs{8|al0@#1)Yt4 z-xm*-5W&#atj04Xq}#kY&m_%#aS@Sjsi)E7dxBg#nwhzLdjaO2yl=!$N*ar_O57#d z1yVPr(FbGn5-=JQA3^A8i!e7k8NB%#a_P%J%_eq$J=E-HLa21DITlao8* zkuFpR>;FKka(NM3qHQ|EjzU}vPBT)c<6tJt(NGdbuD{k5MIc?sq-Vo>g^?8#t31#T zWs-9ocZG3g8Ka$v8JScwZYpC8VnV&#l|=%)Vi#|gBo5N!yI1iq?30SN^k)tf?Q-&@ zWmh`HwuE`yNP3Q14O@UAfv}@KHWugV8)j){@2InZiyMWwRTv~#4VFnTne*a?>fd>F z8Ay}GyP`?59Z#9P*YNE;S(K@oCaxz1(r(FN>(e5<*5j{)jSt5i*&Ya%44@frmuckt z0~+Ezfw`@c2fc~E10YwqCjVI9WH5&NsMz;f2zLRQO|b6&Nr>aJIif)SEdjI_=n3)r zA_NN#hCR;+$lZ_myQfAUp5r>O+~YWKBCZlQ))MNF@K|NvQz5x=}pt4=zkp&YxdfXce{{e!u>%801t8Wi==`3lDTk!uJmQ7w?EGfbq9AWlt z#Vi4iAg>5=d+@X%vq#cywnW;~_Wcpm6&~ZR;2M#Rmv409=05>t26B+^ljZf|Hx{v` zb0fa1$kc&w&~|$M4`pp#9A1#gYE4{;4@Ks|IU;8^^ZmtfF($-8vj}J3ne7*F8#B&U zT8dq}b_Uy*;SgDJMjDNu3i8n4y1yB>mO=0kgTCcxeE(*Gz)^OG@w~p>Fgj$>$M(7f zgi5e`9gHP!g>a?wyXCcbP?)!;uvku&?UVEw;ZBPC5)Mgm&21oLglI1gxSelTiHo$l zoE8oXa!b@uK*vl~TelVN0L^1tE9e`dJrEI4w&Ujs8AdA>xD(hVYoRrOF3o5xD=fEm zUt?I^V?#lyt~_e0Z}S{E67C&ZWq2;jyL8s=zsupd_ges_7g$E5<@c2pCFQi$Mq>=;zI^l zXNGzAW+2)H&@f|ci~p=|YR=2+#Jy9L`xhO!&Mw%8^0tS;DT$0G;twLmk79(9Iv73R zA;GQ?vMC-enf8byG3dHt2NC4SYl1Q7qkNm0NrbL;TuK-@bVx+Ig9tZ3y~$pOg}qCV zQ^zktuZ#=;+#LpL%xXF}Rh^vKAo*o2$jmQ0sSrirm zd{Yh?IL4e$K)9eB>$$NbsS#nGtvdRL;&hRD5bK?V(YRfRY9d9o4@JP6jF4&pOD&fb*!d#lGLRsh*C%djNnof z4OYWY`xbpu0_{qqse{x*i1{Ut{4H&=;%6@ak|yVMJW8Z&t`WHNgulOU&zK5RTq06l3Rmogi$h5PhDaPiN7KiPwg2K)R4^)7D)t-V^O| zNneNiNa=TfA=CxovX5g8d!vN8AXxMJ|jM&8cnYX3i z0aNnYwf9DkNQy(HXpf(ZGT+c)W`+3bUH^@@IrPovd(Uyv|B}A&X(Dqk8fs5oeJJq# zLUW(Azh*kz>jHB<8N&>y_*9Tt123e0-+Uh=v$Q)ef6ohZMCJp8H;q{R10XjSN3@tG z(q+bxJ6rQz1UO;buILTRgN4A8ejW?BvJ8$tF1p6i|KLwg{tPS}^-?~WLp&+mwUb1V z`BNk3ZK0lN*`sU3rs;9ghecr4Jz4l^P5P5iO3Jvb)7bBb9Pe1`O^L<+X*%Z_TZpEZ zC%hkyn?WG5uo`M&!6$_k07$VZrqkhrMuu`aOZ&UhgLi#nn<{+AB;IZfiNqP zveP^9VkF2MC*9VdtVGu-F(&Htj$m9T%;|CqGZv4OVWe5d=BgEc6GmP&Y1SIJdqA%l^EdL%T=u5D1zI+CFb3&_&M| zPrb2|Fei?(M>pos#|d##9AiV&fLlA)?itYq{wcq8LNnJH)4cwwLYl5~0jg9SB zf^x~^`9x>2Z~Yfcb!OtI#6F@NDc!xnu<0elom8^sJwi|(5;S`%XvU@E3G?WV!Tz-9Buky?r-wH zYE37I14Wqm-5lt(eL75#ONmL>K-w)lB+NCJx_m9E-mE!q5m!2j+W2fBh?bS*OINgu za;$W@m^9y7WvoFxS?GPuq}xicc8m}giMLU7 zsW=*E3v&~c?~o&nPPlo9(k!l`DzW$i1f?^zP`gT(TWbm+dwJZ7r$sm+u8B}OWtXRo z={nyAr>``)yS-lDt}>UV=&&W-gk=_Th*55X*V1?98u$Kbid#n**)X7|#JM6|F5Ism z5~{{;O9<^_EHy@B!orR+UA9`AWr9LgVn-3qK*F8$l;rLu#@AY<8upwO&K#Yb{Pc7phG#NI)%R)F8 z&XU|M$6-Ru?deF5yW?a5ZuPEef4nY|9LRv(!E#0PiBporzD)9`!LyIA|Q~mwQHM(fz4`= z?uSfujksa;A`VJmy07G9aHl{AYV^_T_WL1p?wL5GN*Y%e3UinqdSTom+MKeX-YBmj z0BH=aHzDR+lW+3`6=P04Drr&C; zR3ehyqA#_(5L!=*Ok62~X-%*Ca2qAc>>XEFrPDrBCV^`QrkHp^h&4M9lH_}Sb@pY~ z;g5W#;R$BWoh+8Try|%j!H!~Td@34_BYV(cf6WPyZk1lt)8VHfYeGb5Z`Ze zEWDK}ujwLTM?YiwMSW8zt;I1k5yHvo1tFdvgbzmxBm~mR{^*?q;bP)4gZ9L3niSWX z3``54xMqJ_hYKK!3QNn0`p$(!h=V>la*pQ(k}9XP{@8sAglmblPAw)*g?67{t1L5R zEJ20253q$#8Wv6)Kz*j%)`hXlG<>j-c>n5c6YZBdKk1&`-| zmIbH!Qv|_oPW?Q`9={byp*ghn#C0N^EJs|r#}@+3ch*{|lwZ28!=ozP8^h~CkQV!p zUc}6gi_D?%-A(>q3o)P2cIZm#MzO{EfG(8gJ$bh{xrDS6<59afTae3whj9G$Us;Ah zM9GCL4i7gVOdjj_qexmRs|#}>m}NQH$Av=N6k|1p(nT9Wn^!HiR%|>pDa6Zy$9W|c z-fFxl$T2y#Ap-ajw0apfw8!b2!NLZpif-aYP%f<`P&mmIkQbIN2AhxhhY2-5kcCTU z3F*k;0y95Y{k287NDvwE{8HSYZXvUPwy7v@T$;we+Z?`;g?rl2FmWRWfp;`Q!U z{3`#S6$9TY|Ho$+<7wqhiWGH$LFIoeL}eI5rpA;_`A>r!EkQ&U{}AaW(A3L?NitOaw_tNZTX&88xim=>#r5e* zceCEpmn%Y?V^gy94$id+i_nha$~ccK19=A$okm>tLy6LmsUX)LX>%X?KJ`H|@9t zTpOq(1-n`;wkBy7E+I4$BaSV~TS~H=JS7knI z#ZC(5mZ<-zk#8svf};raKpD3`-vql>5eyg}Sv+nv%%s^D(>E|E)Lyexq1`qLw#*@pW-xLh!gyF=?8Y%bWzvxfG?_4`7*Mu@l(LyyKj zKQ_ep(M+J@^PnKj4eYgPox??XYC+~MW!k}5?k8XrMQ8VBtRc$P@2m~U>S!||Ztfm> z>pyaEtxLl(aXv*94_3r4icSJ3iKvcX)pAK{xN5{ zC6dxNz2{Sb&IC!7#0Od}7e4^h{6(#fRBtH(ULZl)?Rf#z42b}n@>fMssOW1{P{1117alO8Y#nOSMPIufd+Wg_|5H9oLzMq27ct}+HV!jH58LVfM zIF<5RoG>?NjRk{dNg|3P1e56seRgbmWA*JlV(EEU2>Ixw!l9~6%8q+kk=N=XgvXx@o@kKRPLOL2FcQy-hSPFT zA8(7u4M-PGiUMnNgOLlvhADqglta+ln*DY5o`(o?$uJohCf_S^9C=tBjQd5o$yntX zE6ddZ=v9*^PKa6h_UVCQ=R<-9)cyLyu^41Xt{)cQk|WF+igTw!Q!plL)|X2}QIDi_ zyG)P&s-@RrH)#|RxopJKh-f#o#;NB2u2`%W&=te;Es_$dv8*u4$pfabeEzO7Ra~U+ zJOQvvL5YRH4+!)WRE4jbH6dNvbT#^{XfvsXsT*t=>H~3(x*%c~R*fTsxxwH!qM5On zIa@GvC9t*SSyW-U04N35lcmauv&H2)LyDYN3eqq_$tE5YVTN)w&rsQ+9{>gp;%K-d z{ws>GoG0jm0}#$Yx4kuOUt+lypobPWQMy+~4HP-%3BsLl4-QWH;z9u=)gc}5?+iki zFE#X?*AdBo%@($VTZnK>8ip?YW&RiW-^93LsQ9<5-i`~F7%l)_{ABVxT!4$oNkXD_ zczA|DTCEq~95$7=xKV&PrCY}a?iTR>{4+r3 zt0!EgKHjbla>>sNx($agvD(4VPQR<( zm0o^YM^LTbk#CRXwJOSnYMzspAwj70*=zDMVlqo*DgsjNsi7ISv#b3 z0r#vx`VEhd48;2)T-gpx3%FuF0>BJGWlb6qF;qfaH_1s-943fv*xep~(kIQqyoCfi z_WebXU}Fe#v$*%j0_aSV%b}wH+%x5Ejn!z2qn!l%njS{Pp&}_D(r7o6xjA)PHe&)7)MAFx=wqnsZK$O=j{GVWB%1VMn`Kg%>G*Fq&y784L zbDFy=32@=yqsIc89aJ#QG4Ba7^AR8F{VwdYj|0;f#ffAbC&F9}h;;^li2xXRX9$qA5~Uw<;2uzm@}}OLOQAW)Z;L zNUDY^(u=~1K9mI7yPph1hUp#AqeReq)|nE={*q7USDC@Ib(3hyiBTwWhIm#a(Rc&6 zJAQBqgt^f|RgKKQ62jkHeX;rc6@jg_FSCci!Ecf{%}INw4v>PCPY({ z){8h>RAz$S;9K%EXs0Yc9M#xHK<>hRG-tmuNPm3>=j_V`P&C>R#dQ&8NH?b|9!QES zP6u*>tTbst+$z#pB5_F~&DdKs@l#^cMR=-dO5Vs-D?RcAp*fX-O1gi%QIIRz+rg7*jhOo^5DF$uGwr}* z*$gmv(N*Rk`@R5IWu-Qz*z11{?H)3I42ypYgWr0~u@)Pg1LfhV-Hvu-QmZ&fFhy=L z@5Pu1`b5fab3efDg9&6AA#Q0NSU?oaY>UjM8fWPbk7?B3s~W%fW}Pj*90~Q# zjv#>@n!#OG83y61&Pn$OaVOwWiOK87O9CifUD`qmo?HCOxtQ(ATIT^UsD^l&c4K`L zwKd9aTN)e}3m_`5KGfo&{I3|uW5Sh~c0T{+*&WGUy6)IX7!1>uRB!xQl*`hdOk)2b zz${@gm*vZb7Zi!0tiuKp+3BvrC=+s)Xy{urcvp;-LC4?PjvItILCyhHo}5|vLJ&G^ z)~nlJqysQ9%ILl+#Jon($+=il!C;QX4U{fN@5yAUYb7@REm5cyjm38Q7*Vb_x)D7C zcqGFlrO1X?L?l&zt_;PMxaMiR&s^P>YDk!I{FfSUGU0NkT*Co6Fb{Heyl!rVtY z^iJ*ZqFtLo6o`=vTv#G{Lz8{SMM9hjwWHVVc?kdJxF@FSH|2(53R*Bf<2FP zc67utR}{J6K}9NH)mTj!m6c7D2E`_27`O zv7OkwUZ8I?je#X)+*ivea60?y-R4Sy(b+Lk>55ZCI8k&{Sx``bjlT)>bVTOK^$9^g z7Mi;tk9m;%r$Wraq$La!V)LuYP+0NOEZYciJFvr>vVI&=0tjVsB>o!#Zv93DUnJkV zx=1%&lc=B}%(=^vIbwpvgt+K>no0|QH7+iLNduJzxmw5)^M8*nu4ZPqs;!a~X|8Md z%lYDvk_R9}!%S&<{ZZd;#}N$2ZWoz*Gx|w`l)C@Gw>uU7DYzx=ik}I}Y^CopQGHn=yhx&K7WpG# zoUi0Oow02R@H_&&wJN}=P))^oW_Bb=#@{R>(GlU`5`Q8R!x7)5Ou-XGIg=jtkGwX2 zBLD>_Y`m`TeB#IN7)oXNUTBw_90)ywI9}gQf&C(){ssQxKPOF_636S?RnK>6aoEj3 zZZavn;{K4m*MkDhQaKQzzl_EU!Wb;Lb7 ziLZ1U6wQwqgq@rS^Q_g)y+u!KDa3h`0)Dv=Je(=aLlMsbXmPQ7<9V^$b3%Ijc?}V6 zacmm7;SlYF8x?uoh+ZMCfK=l2#=@*J63?nN;w#Z`TOJw)?wBosWMMZ`2w5MNj#`q@!`FWC?)4O>plplt&8DWAJ1Ra;x3_P2seNz z2h`$KK_2g2DERa};6$o2P+#;xh`a=JBdtIli~%T1uyAvF1RuHDeU}Jsxn$5WQwY^a zeR}SPAaYOY)$<&?tIt%GkrS^#YGn4~@x;6kEDkW{%gc&rFh zYYwB*vttQpN<|?|02!iLI}ocr4&ef3;y!kkNgHDG=Lqcrwve}SCAjHd3j{wR z^6FX<;%08qe_HXm0JERX5#sE0;(p|LP@0WBDyG&iI1XB8%+U1aI|a}dO*C%WUgSGx z%38y^{-}^VtJF9iGB4fy5|9~DVaJ6;pUGmwmjTJ6V~G31_eA88R~gky9LEikYLydPz=F-Wc&HG^_E{%9V zfcS`PcAhu*&J18K$2KE26Xf{qNIs%f2B9f9z2bONkQs|ziDpU4w%VHoNKHBX>C*HN zfp8nGy{goE~UtzE+@_%hnEQB{ST;!rTGrP<>p@rJie9p=GlfwvG(!jD`7!`km#CX^H}f}PPDV0jM*bHvv9E|6RP*aV z+>rm~u5~nC6hV=b%dUZVIseyzvT!W*4*!)hjHJ?>_L9r$k35eLVf4yeD$S+U1iLRF zszV^XPMHAL&fbhBt%V(C3pvK2Cw3R$QtDkEZfu_~gUGicj)Th}vJ_*TcvA>vmoFAB z2Y>voNi{Ni;Zc!htegQS@$Cn~$|+PjCcFWaLG5Jf(zMmc+W0*}x{8JW7ZM#2?0zAR z>4BTm1-aJKlg8iK0-OoH8JJttfBzwnCS@|h&yol=79Mood7p2F7uSrfp;+SsC@02^ z6RGbgz+I}v3o|E(bd#_ap}vDWJ3U6V`#<=DBq!4EUeUJ;KoKQoV+Q!ahk%|yq_b%w z={QO#(Z&#p-=c3yqP3zDYy1<60<+f`jBP%G$fH)y5?Z3o9dS*k05#FDV7DU9x}>dk z;m07Z4-O1if&VE$<($wK`+veWS@g=Y&_^QjNachp4T_lO(;{|vWrSO%?+I`qTo%gZ z{b+d(8u{qbi}PPW#Z z-p8|*p9wJ&kZSbB(Vs)RFkZ1p<5Xd0eOiNYg{>-9$THZ?+6u;NL(_8}S`Y+?fG&8(d5c#IXV#360h? zdw(LxMdrpwr}Yj03qtMsaC5zzzVnuo<0RSy+G#N2g`2@-gn*e=TljxqZ~~3u?pR#} zgTvm>V!m%|J)VnwuEe53Xwr5bpy-LIqMROEiE6S#*jN}z;Lby3zf6@UZR2CFy4+S>ch!pUzpt<#ar~FCG&BH!36b zxNa`KT?AD*Sw-`>o=`Uz{Usk8ZV+g;PRD^RJD*$2U|LQQp3yz_nWsoG?E~#kswUK( zM&735LeVaA*AVY(2)8R_6nS<6IJ>&2o&W zJQLN?^VFOf@0K7MTpVR|n0Qa1JB?nRV9ORC3(B2FN}Pl7A0aMsXCJbgwdRL*nbZaP z;`aim15<;#=6(x8=1mhTQmd7Anx6=zh+VuMfyc~+OyPfo>2@Je&NDf-uEsBfVCDI(xyYo_-8jSx5bUBl{ zJX0!zaq4$L-Cen4#?1jW`ca9o%ppMLOuDTw7eeldC73%`kgJgdgne;I8L=jhW*UQ2 zzDFpk(vB`*thz{%o1_D2(K<#L0m!EjSBWsQc^7~a{wMigy$T*%Eb5?`TJcPSh+HDP zCkXmLP#(+pE=*(Z+X3ckrzJ^hUaikw`gN3D*pfF#T z0i-9AE*sbSJ^|bmFa{T1q8VlVxG&m&F}}z9n?_1kd@jftbAZ(}3HZas0iCfn?A*3C z&qb3c{;jwj{mxPjsbzYqzCEOSF~HbQWUdP)xUhl! z_|jl*ReAZW#jgcWDDJ@VRk92O+4i!-++W|W4fk2n2s%=T3&LG3t_ejutt4KJ`IjxC zbg~m-d@U%%wd9P;%fF+sPzm8^tW&$>50)d83!2QE)exTvb7ovpVa~Wbbl!qU-8pN| zKPfnOlyOz{l~!<6dQS2qb(LtZH>@I9BxFOUKMMEeicM&m`+ahJSbw9$N}t)mG)%XY+s20dCs*$KQqjY0zK3?bn_fG;wUU^RxW~~@Sw9i z%>T(MfG#ZOD0J7@B5${9foZ?Tq+N^ch0!-ql#G4ycN=!5UHYbg-F0rCju&Chb;zuf z>n<4lQVe2z81oDV|hVI?t5 zkZZ(|Uk$fW83^H8QfWO=5b0u?7C&2)Zxswjd~ux!$3{XWfmPpHAnpfrea75XC7|R6 zW5czJ5QsnsF-w3MM5FNg&}eKY&;t+VquMT7?T*lD`UJjdOk@lkHQyHHs_0sv5(~E#QRN#u zHW8f*KFp0yGWcJ7BB1%sdCYcTs|X`q&hUuX4wwW*j`9cjq`uuEoB(iM){1up5jeGT z%$y9B$5jKp>6m{?0dTlx4NZVck?xk5Yq{c8s9dxetVT(NNeN<{Nt=TM*CwCk?x00`WRb= z4TP9)t#rEHN&sBfJ0jPWfA+97tHdV)NRAV)T%>$a{*Oiq-D>U)iuCpL71P2oLd+B? z!_&8@(pLg=6|p;r$2Wx5Sm{F0x95)_T;E|1Ns^6oFdr65YcoO9BKK|N2<+YKan{BV zt^&&jE&Uq-uBFBrJ#fM%AWoA*UU$8{F7wqjHU)Lru;N7Gq`hkbDO);vF!A;&(0Bvl z*GF5B6NJGA)*sm>+_4#?^JALqWpSIdIS3WvspeXYh;aD^&+K-cezGT33)0pnvuxd%JkG42M1)nqMtL<}(X_dwi}SF6eN5 zKw`T&wgh28>l}$m`gGvB97FwBv{zcLY`bwJo8*E^Yz0i+QK-{Ne{YerOJBz*^ZL76 zLpg)w1gkwh6_UrL#CNgVHUO|jn$+<-5oRtEHQkmJ-xoG6Ku))rj3h=)YzywRaSx)o zZ*3vs5|hxM^zG8?;k~$DG!+{`_OP9UO>A2?D%G_G-*tC?gs5Tp!E)? z-Vxw_OOYB{Ll)c}hz9FpJG+Fw^SIJtC7I4`!ra6|%umQEAKn85wy`flHEk~lijGYY z>F>5Tl$(d+t&I6$(McuPo}}A4QC=$0Mbp!c`0NxwLP(3*I=?5%9X&1WwPtdvvFtv; zxuu8NyI^R4{6!eW;QEd@qV}?c4jXj%L)pCdl2SZ5m2S zpZplwwPjDhRwX9?1jJRU%1TeXjwu3N4ix*i#X!Y%YKh6`zmDoLO_+N!CIuXPG@8^^ zV3q!I&Rq8O#Lq>$a9Qmz4lh$o8dGBCRl>|SW>Kc*dG;&f;ieXCXY?ZlIcbJRZ>)Df z5d?8LC&*@eR6_C>!84w*!GVMtZ+w-L32^35!N?7Pp#%-BI8gw-Wqq6;oThJ=7)Bvd zZpFER%s%WEbO-#X5E=xX77fcqDvsaJ14sYY&UfyeFq_B91voZ#EnTQ+eImrnr+FKr zBl5Msc?Td3&Z!%-gsKU~a9dmL5 zA!uRqd%+;^Wu#WV^9w2X*(j{SO!?eEsp^Ea96F(P0@(6~R z=xN~v8UQXXUfXqx&9bzVKo>%H4b9k70Nob1tZ|9H%_X#1V~*Y;EM6iwN^dS87Z``8 zN%U}~Fc+B3Ij5y+yeEiGgI7kz(;Cep06S-1+G#{pka^ZUqFSEFM7xQT=1)2c9pV(^A+-@p917*0k+$;53EV0I-3n4Ray2TM|zZUGwRQSdadyeZ51EaX?5V>@laRijp)j~VKzP<7bAXi_uHF2|O z+6S{Vshd72(rI=en&vdqz}5iFMMi+v>eeYPZaoT+jPSXS6lU(Dq1*wHx3Hwo5a3qf zs4p|M<1&Iq4Qsp@bFmOMupJ}uh(76aY>682L?*gq`?Ah4{G$p<;VaG%<)SbSS$gFR z_;G<$K^=uH*sCH*i^X~%Ce4H}2RO>7z3e7};6R=~<5bbEae(dIoCj9lRfGyR&P z%rm5F^;r5iXqO`CtFVvZx?7-G(4QRa{X&3Q!1$9=-b^8+iq;VdQzwXWzSIq=4EgpR z59WEB{h<5?Sxt#Iu}x^Hh+Z)_%VyhGuZMq8Bs2y;eBt-Kmnom52X>BEVrNC#k3pnmy$iO8~r zjoaHoT#&AGmoV?F0vO|-w<{JF;H;%{lH9aU6*j(gDAgGb6Fh!-&tX;aOpJg=7loQ@9^%qWG?ZK!rnEPOHqqfbg|zty*k#;(&(fL|kw zzzv;#pAg~NHL#iE^^;!$xPRD)j$LYOEYQ3jM1yO4k!}jy$PdI%%Q)0X8w`BX+)@T& zbsmZjOPa~772iIEM9lkPHjRv(g?18BP;szKkTL;?YP@ zZUuxe2m<=zbOm$mS~|$bon<_^Jd_{LmjqE6JtL@Y{F76m&45;q-agh@_E>?=PnUH% z4Xkn+5Cej<5u1;LPKR>t)V3}&>8PFoW;U@as>GmZCxR{+YZy0@We|LIU>YUbOc{_4 zM>ReZLTgS$E%!TT@~xRhR=Asr;Ezi0P#l;4!0Qx>a3Uxe1*QAmB+9!paajvPx;b3+ ztQ;!IJes0hRKy6a_`PVCgL8s>q95>U5Opz>9C_!%1eXU~5{htGyvvFi)pJ0c8riXQ zi*_^gB)>ol{|3N?;!uwl7e=e@WqMVGbzw1oFF%FDZe}XUYUSvwKVsot{99ym-2d#!ABS zEm7I00^#*^-s5e?M&}iY$VZEgP4CSHu_=}tuC>o6l)0mq2V$ydm$B8?>2wb-F$R_#nF7c5s|9)M!7_w(ae7Df<}zUSP&?CO zHEtC|6SuT*u62tnwq1iWCjLv(SND9u91$hN(> zh&$9oB^4!kA!IdzpQ%{wlHzZ<5Mb^&K?r~IPBFI;B0Ohy<95AOo57x1@`yDqoh=@R zr)FFw#Pig&DJ#dbB6I5_AwwSw%l6B_%%Hv&Cw5l9Z(af9?kP8It=LF_;}MBTmlFS2 znCs8(kSl9m=P07FJ)^JQSORoEqzreLfS$eNp$q}k3)xOzOuiDr?ZWvOiEV0^p9!UR z=+QM2iMK?%yy&x3YrV~w=PDrAvW|hrS|Z_eXJtnGsQjy!xowfF`HTE`47w82MYx*m zs+rVI5+ng@u8LuipY*o;u1_5*+?b{>x#dFHg86BH0X^@ zg_whA$28KrGP_>uSS;8{KDe7`hv5#8AqJNZ5~fL&tKNY)T$HmSu z5j8&W7@!ojJ7$z1?%vqeXzlo`K=|FoO=~Q9J%qDQ7Z4YSrr9JmXG3*^NVq=<$7xsJ zz_*)*-4>GiJ^lcYd84aA_6c#1Fc(8RH*1mX`^O??+Vw|8fQrs(6|Hqq6sTAIbEQ8) zC`a1+R(+Gf2+x1;UN!n~U2(>ZV9o%q2E2ZyLr((TwzU};AGP9Q1#_z;Q30D-iP^6f zOda&dq~0@)KMQdExf^ba1^*09_HAvGVnpAraA)5ru}>0E^uKfw!Fuq`n?TKm5gdZE zqvcvan6pnCSSei7*n0_firnwA$-k)t@Z-s-{_bB0k*mk#EaQe|oFx!;;kghFy(h{= z<`6O#ABlG5F;|hY%bYiZ@bxBuCaDd}+rovS##@bN zL(*9PKY`9fvUCJqua~J{_Q42xuLLS1mKW=a^1>+gbc zh2+Ujin;%o4Pvh?q1AWpCX_oX3bp8Lq?DEr>UwgA#fcjR&k&Z=t9KHQEeCmHH#I4=2s0oq^8#V_cC4xaq*zbHcYMW~QG z#HUNcbW=^2hqnkb+q%%!;R645A#@i%uNMnF3`yQ7TJ*$$B0Ly4PV+jsXvgM8SbNil zg}CY1Kp>Ml`w;*oN0T50S^QB*YA>T5F5@)oJt)Y{so_kApocH?n8Q&iuK9G*jay1c znX4bYFNnhNKvgA{db~&nBNy(;by_-ApiA5?8R;@lKvQDw<1wrazD;?)Dort zRps1*vd-U}2Ii_f)ECuPix6G7R@KeQs30ek zwt9VWq7d?B&mh&>dqh%KmJDtpMVL{T7YySmJGBrxkl*to07x{uh&Q9ho*0;u^`23jY;l5M-dbetz}cB75{glHx8Kh!fuf z^@zbIpA?B0-4_XTYSSw{^!>{Om%BfX)-#KxHJ&ohu2`{MQu? zeZn@aw6Xe!MOZzrBiqQR3o~mNqZs+8Jy1e3|NGiWG@kuu5qE$|lXmKecZB7R(TC>= zdeWqifHbX7m=s&#CDMI2LJ6=+5b>tWla{|n7Gf(4CEii5la6qdVRgY6c<(PXe2?g)EaCO#U5~{U71n68NXV z#~ypHrZD)?i}3I!ebXV+i+S<5DD$$H-nh>Q%65e&ew>f4jqj?mWj_qQM+y$Z_&B=9xv~}AiEHz_4t)gmlvsWrkw*0RMTv_ z`8-9C_CDJ4e$h;2xcmAr|0S7%KbPA8wyg(EhVWiwN zo-C3O<1Yy^LOv0cCyi{sz$UX5Z+{EieRfc5D;D_kIWj#5Z4fEB#d3nEb_b0*S>NMb zCOzn>&FU#RM77=%r_Nu5z{eRDKy)_Dw1kc~oSaN(d|fWkRY6Ho@07-qg4~wUJx(S@ ze-)OQDns6n%e0ch0h2eT0}B*^xR&Q-Ytf9fE(yOs7U3?9jsW6T3H+-O-v;(lgcO3a z#L5DQgqO3vm?DB|53}M9>KhKDealTi{H~1HsC03~_HYU4J-^Xdb-^NO^cv9h*j|7+ zU2CS3%!5K)Ei6EKVzGswU3E@2eQ3fp+4Pmm*)8SzDf3Tb9g~(=o%nMD5<8tCV zk->hn5^+Tsl=x98b4q6)GQx+pK=)D^Q_zt$nOrV3*STN2@IMNm#FM5@iTm|!?n@CR zIlNk65is}9Dw{0Lfgd_Zb~?Y_@HxlnXRo@WH8%qMX1nLKv z{=|WToGhn0iW;+oxKH*VCp=FiDY9BsRvE}%W{xcaTVJqv~5%u~z4-p7(p_4ma9+#teC+8?*ECVO#+YFOLiOzez5Upl? zQjL2~6itGtxi7xGJOp{QwM~ij^-a^|7oUz1$pY0g5=XDVrxK9-DCS-f%0=K6pE@hA zCe?X`x_6-Jj72?<*9&q<+HsM^ytastaYI@^ZDJL<<@E={Tl&*5J!WK@<%o_o5Lbz& zoDKX~t@8uEDUFUG)i_2JrRG{oB8xRwf^MBk+SYtlRLDH-$@!tlk6hU+mNV9w_#Hr|^6;!cTErM{bC85%aC6eTt zyjmc{jMRYgRaY#!S`m|zCGX8epAeV7%2VTUqG*q?%1~!qCX(vtKoLudTcMk zeUWENeP7xiN|OVHt=w0I2qFgt*!$oZ}(w z&jMUj?%x}6muT7wEm$b&Rvqb$iYVv6SyDPj@vSw1oE{eqtl$Zt-I*<7G5yK+ zM9vA@>Dx84GQ9Qz=LmEqBJt@S!K@LH1HpCmYYk`^TI+r78P#;OCb7qswvb^@% zMO1E%(duHlT1S}o6U@ytncPIT;%9sX z;9RC}GmZ(GM{e_f99m+R2}NSaF?86O&<5frd$RTRrPz}{a1dOf8Q+fF8g>zTT$T;EXMMWz7L(R7m@2Z(T`dAg=0-)2_` za-$64+DtCR@VZ=v!%AAFCWc&qu2CNSifa`n=U1&s(LG(t-%>_pV=UF8xV40ETY)FD zTHG!OE_S1}_2g8(%|#rxjl|zYtIM>t#a`3+CRgj%atyjrfF?p%8c&Gu%!C>ZyAcfo zDYHymmp@=$XN^S!HQ!9qghH5?`aJ==C0RMn3 zji3RIuH7)ls(vAksJS|MZjOTlI`O{hSez!3ZpCG~*W!!I|1s{2w?%puOc#qXJX&Nc zV=_WdTF-Fq1Yxuh?i|MA+O3O7ohW}M=LmNR%T2{bT6>Fk1-ah59nup=Yy+(U!GQ_U z*tSsSYu!G{wHr#Pli^kbL%Di%DWGQnRN-b|A(S)#`UIOp>;mdc0J9-Hq@`=f%M^xI zLdh_uZpSw@Kmt#*i{Ot*D6lWReUT8R4d}$hib9;DE-7k*a^#Qa55ca_NO!#}-WE-Y zy}WxJ?}{+rI~WZw?co17q159e5v~-*;Oty^A8AJ*^BvtQ+6K|s(!!jr{J?XxNtZ^e z3C`K-renJ@YF6Q9-z#jK`*b2k=s_ja&RkyMdhu34l(Ef9E{p60=@v-4c;(iLlZ56L z=pL$Ssk})L73I1WYw%qlJW(YFU2%K~Fex&FEWT@*uBB~i*}aA49?i8P$o@h!15cb7 z6@9xe^ZW^BIXW%P6qp!HPyM?Hb4>2L25QKD-pc`)=x~C#o0H>J@wivtc{b(xq%Xd? zI{>-Ml>x^#`o`u$ayEK*WC3-;p5 z8GT4r&B^fh*TUT=nhcxUM7jp8p?2O9m^LeHzP7iz3w4_{3k-4Fl!SYt*h{B<9!QoB)Kop)+m84R;2^c zhzw#jRY2E>odLEXdUAwImB)qCA`(u<%OW(oDkE6X_JUgQ99crXPoKUICZ0%`xJUVoY}58g37CUU@m&^dMKp&-#=%$+F|wql5a*y^Vc z6cL9iTF5#@@*heGmH3~CoJBWc5eXQh;s8ioCd8H^oc=J+!sWSgJE7(Sesi>6+ewHS z+mFHxdfPdbVRpxHLMFQOM%9%f-09QKw$6*bbg&l%kF$Hzh=_J0rbtQju~rS#3_(H6 zkGHQa%!#qY!NSW+KodE776RN^=t9hmYWNL00iC4|$GDuntgA>!jn3P@bpZDQ%r3Dz z>*_`!408hE)DXEf{=!8#9ds0VsVgoJ;!c3vfVD!jv-jiM;?jxJ0U6ZngS&g9s|U*5 zr7zJ9Tk(H_97_*eVCO>Bo)Jj@>qdrjWP@+_ZHxfo6p?v3;h~Zad#H@t1-k)cQ!fKQ zT64Z$(5%4C@r7G+23E4YSVjLhyo@kAbd^<&RfW2e-F>)gsKr`B%=JNT6)RB_;3}l! zHnFyE0=Y=(BA0{c7@6-Z*jY%f<(KDU524Nil}m{ycI*T3;Kx**=BD%vGW3c2`2*gi z{d=q-%6#ZW>wSHZPQB7E{VUEDwOHvOP&Z|J-8=eput>Ob?H{VOGtDK0#+6&T=vOqg zG`EIvnh1A3mL%IDPNzHgt&h&5KsW1+r%LPT^wvoSas7mcbO4TdQIDU6XPs> z=a~RGLoJ>Y;NFUNgTa_<1~lxzIXLatLSE_ z7E?x{Twv}+FpQ4Q65;VTIU{*qh#4x8pF3b@e@D{|`+I`jfl&ELy7pc@RyvqJ@`R9Ncxs#k=0t|t-SZiT>y%+rzf2R@ zT8EH;i^Rg9mHqc6!ebv?{>39g%sF&=xM7_CPylt?v^dXo7!-WQag^@d#zex_Vd)9a zeYgphuz^@XM4lUl&{$#4Sw)bW8Q};6BR%GoP_wgD(ZjT~?&=EZ!NFx>Y*VHsgPG>( z0zE^B4XN!91(|mf4-M#=w#xIiOZGYp_@vC%O=x!|n!(!%6b?Gj`Uko8xd6G)ez zIAOiGVkX}%Ae!wOajNfH$AY>e$$1jqUXm*91%e%zX)kRx2z<9tmj%THge%q9=ePpG zI0!c(197Gx7mw{7My?2sVx!{$H4<^{8Rwi(MC#{~Smn4v5dZUX4lc$;Wd0Iep08q; z6TvikTZ~S9n~{2cq+-AY!+;NYHm&*gF3pX()*-eD2&McA+s5qMu=wy-ENGaogLKs|j zCi~r!p%xx`;Fb@>!wqsLBZ>YqDf?xo9mS(&zf&ya% z*TJ=}?W;dIV1H|m?1$(r@%{6`zteNW~;ctl||hR!6EYt$nN z@hs7)J*1U$iYOOd_u5I4wdh#|mZYze$Zjocys>%uJ+0B(lmU?rbD*bf_7-NA>xD^> zkrI*Te{5(D72-~SN*7zCEq@JQwlH_Gp+ZnB$i+tMWLoTSc0nUl7>dV4=k7_tQxE<~ znA1hOokvTLJO{w(O1}jgj+peDA|3R}hT>PEVIo>{eQ~LX+>i(}al*wLJ?DaXY_Vra z{slF*7CsLcHcE%J65EM#X)!EqkCy0^Dw>Bs7vW+t72@jm4FS%E(VkvbiEo_`M#FL* zjOFy51)VtX#q>3;j;**&f4Eceq$`*AM8^%u1qI5A!w^aP#o5qtYc~2%LfwwcP8vJ# z7i;K4Ax9e_>`FCmyD)O59pPND*c86AuV-YwLJ`lg1q{1aW222t^8URSC!l zxE0%9Lce>RAn{ zQ-640nRa@~PF%;h5@$n?u8>wM%pVi(tXj=cNn55Xq!F|!q?B`5nVFvAODssDp(Dzm zQU$>l`A8vjk6zUEk)Mk4V&BfdeO9Ea$-2~xqb`A_w^U{z5<5qP|HK3s|AeC5ICyo( zl{Q0a>ZM?$hmNrPf^BwL5rmzf?r70c6y}B=-@?FuZRPifUiRG^ra^y_vzbdvIUc)Ws37=4xTPrV%qO=N{~(yIGg1CM)+B}6+!Z`#qtR3Yx-GzRy093iT4zff0WpizV7+pza_NwlzB&@hg3+4LUmTGAF`pL+PE# z`egy;OHZY{8SC8#?UJL+E1~|@C8FIhOaFZL6UHTIH8{4VsdL>D#NI4jdgCQQpc%?5 zOA<^~2l6V+%g@uJU5dNmx zC1PGdkopWa?OW;F1&}M>Budyb|KHX&IXcSdgXlABKvc^pvVLO$Z^ei(vrUhiq6JXK z8$iKY@t!FmSc36%E2chJ6i#OZ^@BAY0;2CA{i5DmipqmejEtKFxH_`$*E3`Y6xum* zE07!zJp33CWvo^7w)EqoNS!Ob-nis(hn5o$`lsAmZuErz$Buq5Iz%}2Ug?z`E;^4& z#1$~$;}StxS`TwB`Xphf%A^UC;s^S6Wi^T=&!o$=3C-;1a6B3l1vpvsoV0g1NQkS! z18;2ndG}5b1xKZ_8ZU{UP7D}sSl$sumOQu`-+G#FGfDnLxK+Wwj4-0I6Im}-dIrK} z;>v>enV7a)v4Kzz0?topm+bBn$NL`%axo*NM1Zk4w4)wo^=S&YFd<#ZBB&gU_k}pF=5{2;sn0L_4zO#0)&ok9 z2u_w4=BE^@=18um$94!|HCd%yR!MLSd)PDu25GXRUV~^px9u9|4(_q%<0v zd<^aKr_+qef4wj_8yW~K5reP^X8uRGlbcR+vLeP@p8&ZWoVFKtM zMK}XIPx~7255nA8@mlbF);rDi9s4Xz`7|2JwWW1H9i#HEM3K z^~2Dq5?d>vGe~<>b%hziDA#~(aYQ*g^-;RqLqc3q66gIV$@{?$Goeq-v1GX>=al-Do{){1`#%-ty&WPM!%%vk8sFEhusvvW@Row^|FRkfQLjmVQww8FpS2L!kjQR zf@xyBRgl>|9d>U!Hzc)S1WCD~Bc{w#BqP7AtQ({8nlSpd{G7%j--Ix0laIA#EGvX^ zNUDv)f_b6L7f$S4c=XH$aKOPJL4euG!f4CYxJ;nir-uGq7t&_B)t`jA&!DL}*iC;| z&EFTyWZE@^EcIJ_k2ke+5M$2yLGpx`rZGebnhTB)PBjou)Z-`-Br&ONYTT&r+9V`%{H7+qxgb_lOS`qmnAug!k-uBXoKVs?caum52z1vxp}@QmkY74&cqj$2p4)< zIsQ*bZg{q1I^RE-6OmxOKk5q-z|7G3z@M$NceSRn(KvJxwp+T_zR zF#3*2#pr(tbEE276eos7$UMuiu-S>hJl`Q*vw{?p1Zk()Au^(K{Y}t#GvK~NP4kv8 z*e*FiA20hr&;an%ifp}7gHu%c{BXZBA_m2Z3cn?R0wOa z)S`gqWjE{8_@OSrE|*@O(7G5aD5&$n_@arUlg)%UA7p*1<;k7*z3k~BAgK`5pE^pw(o;@1Qgm^5;i<9Ja?x)J;?U50P&8E zc{3he42o2-CByqhCUoZ=>n&a+NU5_gP*$TYgqug|3cBOWpqWAc98FJrq=4hyrkBS) zlL{Z3`}BtcBVwx$j?ps{Hr{RIKPGjsKPoRq>?msX-AnN8D&rzh`pZ2+T(u4w3VBQk z^dq|OiGK;u4h%8)BL9!8^MI19sQy2Kh>}&VNS3gYac6dC6G#>j5CK7gfb>lFUt7E|gP>#pOB5UfhvB!VuX1uT6`wla_8 z-&xxX$MYimx`KEjC$9)G6Ie2`c3W{>0Qcx#`RA>_p0i}K<#0RQZ?U}~=fV-C%pMy; z+*Y;udB2qw`YU{*_`&lMq3zRB@;my$hGYmf(*R7xXM+L!GX0Xv*v5RW9=})uA<8<3 zEbFXaq+$D{x4ybTf#?f#rf&#=!+jNEl$)>DAJXzY4eDaBSW& zFIL>lZ)q-$%|&?4z-?`@r^q~tq?zP=0mUrc$1y_MFJ3Cxg9Gt(xZz1#<9h_U78(O| z{IuET1=Kx|qSC|}g4{j~A)a7rTNKH(?qkayhYNE3vFFtEIVl7t&zdtmp4PLACC6nn z94WfndRqeL>4$!f%88EhErG5R{fvxW72tAOD4Vh4RzUt5PV)GB$Dj~+OV|&N3qA#b zXeh`aR@b6Pj2Re_-_<$q6EZdWGfy-2(}@>OcRJcV-E@x;w-6VbQ^*EJ*uqD$-w17e3(+B_#6|lU5eRT>!2feHTxG zo^gr)(SqIJcH?DKeNdR+W%!XD_v`}Dg1Q;iIRLcwjE))cA;j6!Em6sj?T#_2rpXsA;@o9a#vvZpH^+fHgQ}wA(nfqol-W{6M;I50maxy55!-%-XLZ66 zmWw-OLZ`+ke1B4b`$0bP7)N5!3a~Z1ZGYsmRpsr+QCX_K1v6o^?vKK zKzVA%fJUPzoRERa75Zh0ZM~c;X)*q+Kv$KJTndVO$gV}6yozljL3#z5LBnlLjjxw! zdeu4DX0G%((wMpI2a^9zah{gqQ^>9$r#8{Z}Au!yblP zf_I0QDj>Q|IC{GvejjdQjq|98+@o-{%~RcbLUX=~?ZaWfa2&Ms3mU zB|-71v`u0l;R`?2FIf@>nzA4xXiWmm&_OJ3{-ZS1nk%;!#AaWBaFJx>tLWATmXHR> z7PtF6EXcg6Yo}C8s-9;`G}&4;Sz+U(u~(6!I-+EwV{d8>R)1aCl3NQyB(_uYZ2wS?sL3+b&_6PH*Fg8wrna_x( z3aAIQ&SWavSBOi+(gs#4M9F~yb1#^zAp!}YatzO{`7xezfg*kr!%)8N8)UuZU&uAv)Yb#1Ub9z@e~nc+k-(|TREcFVow3; zlSDF!?;HX_!8%zrT%qSIy0+b>4ek->*O}j#Umh30>*H83#Zret@Hqre#=lIIANdLh zoWLoXn#Zt6Dx2&UqbbV0vVz$nhVDsgagksOF-*_CQqL9{Mb4CE8cv3Exu59=*R0mV zj*&Bf65;EL+tR3z#x+30l851+2{e~DQqgT0@vb0BWz(as9wIm)li_*G9(MHltwDKlZFtq}9i?Nvq^M%-~ zEmq>mL}-p+^Io^GV4NnvJV5i-kKy)M4MhB9E*Xzqd!bx^9kjt*hUzE#09_^6P1u7< ze5@ZReH`gtON2SD^R|XhpP4?l)(?EXp3GpGJ3lKx!qPb_mLA|)QaqmlB=!>Jo`_a} z7-1#Nbrg+wR5Tw<3fzwmI-7A8T6^f(RbLB_(Lpxne(@*15lZnctPH)XE%`6^}X__>1-4) z;9k9;65|4#NEZdC4Zx+sTs}sMM(jQcZC?6dO?q^#FwfQxep%5D@zZN9iKUB<6_Yll zbHSh}pub2fyvrn$C3Q?CJ3fJ)7IfoiW|(a($}+HQBOkMHlmoKnjVid?i*L zcNU$~XUEBUcK(=X67y^238%zVWOnSFiH6*G-x6V7c8{@W-g*++UzeYRbgQ+vT_ANP zs5O39Je#MZY_Z0et7DBeSZ;U)Ng0bHgitgLAX{4;RYUBeeI*bTm{rJS~vk7 zEYJz1+?R_-9Dg(zg+yUVG&>P|rE~hsxaU}&J+Tb+FNpg^k`AZ*N_^`$2~Xu?Nr#PVemA+A+(ymugW679lr zFe=yR{&?oAU>*>M=!9}oV;7iEOCWkq%)y)Syb_w7gsH6|T`yz=PUx0kM%R~@NKE-FJyrcvW!p{|U=xbwj+zFuUh zbFXyG#;7nCf3X`7q|=|XGRO~>X;z`n!~w;hvFkJ7H#-_WW^>XkH)X{PDYb&itk1bpuXJ6nm~RR|4@@ zR#wNJ36UiVy)!d;+$P8wFu!tadzS#WtM*E$bKVrpB8Hy>FQ)txOml-W24| zVsc2k{O8=_t$JmY(K1#!uRw+ds(`V!5KEHI%%x*y5c!8tMroGp8_tS4q<6;5uf#HgU{trdDUK7wMHv4nJ$PbG=;*ftA!j;67Q@ms! zc$z?e1RX*vz9pKENN3$*0}+=AppJ53z?gwLE`Je-^%JsWW^S?G=sUulzkH(^H^~2H zfi6|@V6QQdyeTa8Hg4i8f5*~^P{_1z<9qyGIOi9BS>*PFelbT`U*PW7AMXfq1*9H` zp6^3jWVDLTCz>1l0F*HSI|`OHv8hN(Q0LO~pXk|*m_m6Iy(1?^zAC-lvBJe9FyFbW zx`h#_0DpqRUMV0nsQyE!Ys#(>v15BQbV-pnH+)HNGQUK+8{v0NAug$C z+KhWlxY7R7A_E3Zo%Nh^8Hkw2)co@w@@$SLqm^_3^Cy9>4@yIYjaltSATBTCxfVJ) zq@63&=~$Lr-bG4?h`iF5-n#AptPg(bw~fs0@@PR}pFz!7vF zZ;3=si8V=;j=3C)`f+FVQ2amyKcT5$)3+#73OroF$BExuY&H@-98E|#re6W-Iyc+g z;G2}hy9o4*r7bkU9h3HNM6gSX!*8-r`ja3UGhHC6$f0jv36Pn?+2pt^H>)x4suGnE z9ZGnWBniV z3@`feritT3@B^V1W9Mslwp6z&oS%PP^b{{@n6jrSb@5L?oj{jeOu* zlDJLjkPPNjW1g_0f9b*^88XnlW7|di;tC=uZ0Y0cLbC8D>*!Yn0vptc2V;>aHOfRhmVY2t3L!3+zRAfj zl;`xdAvgunFMc@Zn#OVv6V34X~U1 zB?u*wr$l^01T5ct^Eh9RW(p&Kj8&4NWV1Vf-Lkk(pd1rThUpYLsa(5j5&;?1}a=gkZ|b%Ao6T*5WNMq`JHiy4;9cahq{qopl+b#@!O3Uoz?8=*1y zuVpg!BdkCbwe;A#i)1>vpfs-uLB>YWV7B6_UuQ~Y|8ch{XDU}dI_6>bfVdM7Xe=5c z=>)yC@i^>PE?k#~{9LAdQKnduU z%7IwvH{{`(SE|W21NnPYpqba?ZiDOqZP;aRjrV95yI2Wx+1VjHBa(^_v&-XJby1dB zsVil~_t)P7xx}2Rcg~Bg?*nj_3W5kbtG@$r>(>W{8d3Xw@e125R9Q%~MB~Zig6%WA1IIJXpN5n)Es@c6OEi_EY> zV+p*S0ChOdG}-ExKhr8GOy3d!BXvp<)1H9vhewFejk^#2o;wNjH`t14i+z9)vr3oy zrg`;HVdT?5M<3%N_-2ZO7FUTfLkI}Z&)1iLJa`7;s3-k0W=I3^?o%c(53R&ie>TB= zJ@{Y#rN~HDm9K8s!IS$=TYsbN!hVZC1vMR2gMm?hFxnKE2#$83RSnI~jURhx05P>+Y=xy+iR z&b?SLCD9RZEEb_Y$Y!}6n>^>A*!B*`4kBC@dK3oQm~1~^K#Uz|J@BLvL+@J$^7PUlT&bR_0V~=2AHiyR_S-sy zBLJ*`+-C9)M=Q=CakkJxc#|>UuLQXc3QCj~ahLrA%yp2JnUB677f5-u!WoLqUxR{2 za>8h!$J$Y}1W>E<`!m7zu`gZ`K`{qdk8b);=YcM)zZR3C_+=c^l~eW1`|PEOhB-yZ zxqGpQTo~euIJ(fZe*wCB^ex>?t7ET(nhWWOFYZz@a_yC=$yTpJNR}u+7YM%LpD^Sd zh@XhC95wrxO8#w9UhxcsCyS(<9n+`9srh*(mt|h8XZJJq6KS%uO~%P@@`H=4DaJaz zb4ySb@mjAF152PA1dX2dU2&@*H%R|DdPt1`bN>TIJr>Bq<5CfRN0#@Bd%yaBf&9Tf zbh{icGDtot5N=qbtZ_%4APmDPHy+b-F-%}{LxEWkzarR`=+Exb-~V3$u}otet;Hfi z=0zQA#-yKmU6{YvBs}GCio<@yJAfW-F-hkHC+;bc)Ca5a{&$PaxcQK^HABuG!eFet zI^u2-2ne@i5QTf4_n=%;R)vk&MYPL;Cz8y_`(kflye@aRXo_&ba21lWgW+)B|0_VO zWXHL_V+u4+bT&M7S4T=~?&I!+E!YCRJT-@;W0p@XwOitBjV?1&&wLzf?!MSfg!#!e z%k)VW4(|i9crqR0qQG$IbAf`Q%pPdXav06rHSA)FVp+`NOJ5Up?AFTsB$z z4#%;}0KlGpRL)wFEEMfmk2RujBMhb$YZV+Fhu*(ATIiKV2>WNw0A=+R`85*>%DHxz6pBtX-EW zL3V-zb%rV3n&h_>PNGB4R2IJ*-FNr6NE%f({_&yZAuB(?jl1VQ#hKGR~iM`Y&ZIEzkhx)qW~6O{DYAg#1=0VzD3I^s-0QzfbB z0@uD66h_@UXLNF>Q1QEZ4~ynz{6m<(HOO?$u5bBOKys&77+)4eLnR~=d*yqgAip<> za*MY7#Ht`}3_PA?L8D8G4iE~flB(q;5ze|=(P3F7KK)@Jw+FGy#&BKZrH+pPI&*GS zlk(%}jIfqt0Mo*L*^iROJ-^EJAhC;R=iQxxQQ|frNFARLW3l3D5Ue8@i?dgNJn`Gn zB|IgfEiV;e4kE)uoJX9l5Jq9xjHg*_I9?axPvLr&G*CN)6VYW@3e-87{;m*ka!n>26p_|G2A0Pp?D8ld!_Ojt<^k5-TA1&)I*30@s0N}ZVn{5A!H{TC z^xLW^+Ij{~4h!^5wRD$Nd{uiVK&%d`R5378J$h_$S`d zvlDA%q=jSMwf*yKtfxMsXaCH?y{!%WnS#g*F-CE*NR)*SUTA8Nv)gZkdW;&PsoyRW z6&sLdEL?{~Fh^0`v0&s%`E>!z97dG3LJ;@GP6Ay5H1JjJuy-z@bo&Km9u{Oq_qRvT zKaLBajxEmnzoBP0y}a1O^f*UYmSU!uxV229n7e5(`L!^(I;e6yE`n-vm;n)zAXSH$V98T@#wP&$367jb;uO&oX>Ql_m|Wk9X%J4M`rC1tfGlpf7dHg<=d}!qwrID>1e()jH}cOMus8XS z4^4xnvfREm9G@3qcJghkYxdf>0EhuLHf$g_iz23BYx`BDF`E$s6lfIUVPO;kH-%(6 z{9LA~H%onV=AvC~He4L~#@9C~5ZesLAB0#ew2)Oy2=0X2)ESczJ#r+r*bExx&c^)Y zGCk+Em1g$Al4iTth-Eh~GL#p9baE)H1-kS}@YLq}2+KpsD4|DnuEJpTZNXkTB)4Pi z%F!VY_YvK#^pFVuPLHWJQJPIm2XgySY!+{)37{0+ED(^@_jly?B^XzU z@cWE`Xg~U6#ZDkNm|PQ4imxr2*AvY~MWkDV{*0PXA!fG~C;?KtS46l{Bg9QlUK)=J zbD`j)w6rq{08vKDrq2YR5j#6)#9Tc)HAb5(V)_NZb+lRBuDC?t>0|VRI|{aRj7mwT zeY{YLgJA<#M2n#O_~N(P!|{nOlh&!n{vt^_z>X$v&%Ys9lasx%-YoxyMIVD*Cb(L; z5({VZi`j-%M@1g_7YH(6WLbh56juB}AeHK1d%Wcw=hi!p2i6`UC=_2B!z1X+A{k&b zY1ZOQQI-?ghY*-$afz^kWXy~=N`$#j*46!Uo&PAe2gSuA%)y}ww=l-z0w_rzD!av7 zJ0&}vO1ys?2)7oSFE%oq<&S*2z%UY8fW1PWF^VWXN&pjpK-dl=)4*l=4QH`*k|DCts#X3P-ko{Na|#KEG?1&r(ErEs#4+{M%(b!2nW zc3{+tePeuA&+f7e%2QGjF9`PMuxTg!e+nb?q3uDP{h*xx#$rVw)JYa7c-1hhG1hD@ z(8br@R4cwMN=Va}ri6XV%toyF+F}Q1Mgy}FA(fUFj|+4KSPLpT&Tu>{Y-*?{siQG1 zfi^M9oNUqt?RZsbt(KY6j-(i{tL_Nyy5M52DJ6vXOT>(7V#mgPX=N%|g3ztDERj(h zEp)msgzg}FnD}z}Gb=)rsqvVQ+?eC_L9}P_lbs8Qos@1cjx#<3WZ}T53%f`SD=!JO z#4G6IKQrkSUo5{1zq;`;scOZlqWvLa{iPVyhm{b4gBW0|@v@*?OSXEc8yxvruq-xo z9a4Qqi1XH=TihVpVv@X3*?Y)F`Q%-J5pXL6za`2A;EDvTtYh})3W(hx=K$3>QxI9_ zgYNi?Xbn95oW6d3Hwdj76cZ+fMA0LekJ%v8@Z5*q63NWBfxygXMsjKu zNACfYnTk1la!`0!n3FLDBMAh95=8AfwwT4?#vY%C@^~|vnEq`6&IP9z^e9ZJj|p>` zaxlKT_XHu|nnpD;Mz_5E7l2$$bW_;B4HGM(BG9=dpCyQs!E-0>n9N#=$0_dL6%r4xU6#kaXbyF0qZTBfK=1{B**bDAh zVj3V>HgeC_7fI(1nwBdJHOvk*BKICxuvY zIlF4c#-b^_Vuhx_lKW(UQHGAkwPjZeKmeNyxCQgcOki+{&7_E&7yH7Py&rUL3}zID zx~sqB6xv&9P>cPY7qZfh7e!d`P_|>+gi#3V%mYAOi*B7Jv5#H?>T6W725K8}lQO!Y z5R&KRe&wr}(`dMjg9J?tg*3(xlR0;+2z9}G=t;3iv>O&r6=r>TMYZEm!PLB0^X?i4 z7VqGBQ;jc|bhFLxBciiBX&)_x*TrQTt`+W$H$@{%v$>?^Ej^oARk@i|W9NfFvcSXn zYFsFQ?+?`Qg4z7bCb$`>5|4;5ldyAZ)Zykcg2>PIBRclo>R=G($2eAP#Et^UPe+My zYbFRyQ;&N^I6wAxbZ+EMkU1>Z=85DHoj|hemGo-~^LY8Noc(ViCM&teKinw<_Q zGG|Z78nqc`3!)@g9oOSY5e(x}HBjsa4uyn~nzquhIX(D~!WbFYG}0y3{|c0wkCUxR zoLth}crqNf9menEj<&fH8&x38=MnC0=I7oLA$~Fj{&Bew7a1uf$Njtgg=$+G>z;Tb z)4jSbf~H!ohuW417Np=1+hm&F@`;sDYMT@v;ari0l8@^K8&(fb$(^! z5MZ8g{GjlKYxftKaA67h$n!FQT_`F;W|+qWctUS796nc)M0Shg2l$l|v8o{GiikY+ zDE8DR2BBR827dGo94rhu3CC#3hJG#4E)j#UqB348gdEd>*!?1$14>O)ADFic7jH12 zXhHXY5PEQ95^diI&t?j-j&bY%Rvo~zGeUKxD7Pg|l+FFF7nnr>%lLd+@vKl+rmw=Q zA8bHpc}z-@I|VpFOa2$@jzYUC1MIdDxXpz4YseVhWO$jyF#_QyU0R0YYaI?r;XCHc zj%j+%eMn>Z^f72yL65D(=S5gVdWjd5E%P;&Vukc-K!yZ}ECU>3^J zU7G4yEg)Ao*~$^Gk^aBMIH=3dNsiWCcTRx#liDvQ1HPU~Fb~T~HAS0gGx(Zde?`k` z-C(oId>~4%>o8(SL}nIch&S5MQ&}?ws7tK6z%MZCU^*+prD0{vX=t3Z1cEw-Uh_*K zuB%qh)%d3Xc&mdNl=v$g;S8~3s>H@3TtX^6u22Def{+c{>t8L(MH-WJhZiXdT0mly90N+;qq4``PlVs2FU2a_##xTfHqhw+j1`TCDk1XE0go zAB(j!!F4zTv9X9!=qBR=0l8SLmKgFIadCcw)lDPjokRkE9~DFDMNL62Hsk#`X8vil z@h8D~UCD4cM(Y!^vlUMWFFFG*I`NVaH%_h2=1NB=)mZstP*)2p)K+XF8a7~e#e^t= zCQA3%$BczVQjGl;#4km=kb}DJoM`3ioC4&Ejn^l!ed7!33Uloljal_k zIkarrL?|tX8<})jr;Bv?b(W79ZTAfjL~&l%-1xqp-A`m9&Ver?bfrLlJh@eCZ2Pe= zvxiVc=moUfCi_=~Q!_<%YR7WlgmO(R$msF4`=qayo%EUHJ;VnUneWqY&Bwzt%>~KEnj6 zqgE^yk-Hs(7A4vI%mR^e6k)zl2)ydUu;6w*XSO6iTx9irVX8h$)`9rNSx!AZI1t-^ zi)V9|4WmpT7-shtD521Z`ArcnJUU;1a|O6S9Li`|{+^J`PgW7c!-^|%7X7W(Xf)2| zcd@R&%|h-PQO;0XNba8bHh@d4$%X(EBXPnxU@jL14j6jL7~(#G{!X)j2_`j5kPDNJ zu+fJpk&$kzbM879#2>`2fq7@8^8hT7avjo*rqp6(p)Lll{6o4EIF2ttHO6PGLv++1Wq>DpPWCDA9U4U6YbLgD;Jt3CJ9*zg&;0vMs z^%TlY{pp*+s01R5ZtF%-&VyC1Opg-cJPwxve3kDOc{mh5>fgD-{1&!uM3Iljg}887 zqvE*pv;aQCp*d~;;zdxdvKI?dH@zaz^&3RpQu!XFn~M{a#&|C}7lI!Ponjz?v+?FJ z=9C1ZN_)vEG>M|ig+KQl$mi7#Eu&lEo0KZ+(8c7!{cSzF3RR3?lG6PNVHA%A z$ozO+1jR$Ei%C&ljP7wMAm7hUHyGc1O^9n@^Ac^VPcA{cY*`G+=Rbk40%r*{)vqWC zWz$_Jl7Q7OgQl$<)0l&qE&&t@@<3oBjGkS63i_XiTE!zwwI(LZk1o256J;r0P2(y|~c9~Z$cLj)Bm{m_b zHSt)iBa~_({)~oiiL%IH`$s(wyAr@hHOxpX9_6Z}P3z-V@xxRDZ~?f!u+G(BW+rn@ za-!*r?jHkMBDt+To)pRVbR}O;yele8A_}_{5qphmz&w2yM%b*-HV|wE%-1G*P_#r- zo(FO2PoP|IR#M37de;`YFwr}{Cl&mXV1E!5wPujvXbUqdaS}{}*X4r5m1IzNn<#&p zz2iW9aS^mR$su}+tDLmj`uufeHsdYYuNhw!l)t8L$>He`VP+C5KPjs@cbG5GIZ9L+ zQzcZF3gx@(P~{&Nn_ds?N>V2V&yxkXV4R{3#ZPX4c1?&+pUmtx{wa{<0F9*vqvwVA zCAOfw(RX9<5(WSm*fH`5$_+_$B+5uvd0wC^BWn>hox|~ViB(#xSu~}sbW@QZQzl^+ z$D+H0I6qDXiMI&yCs0Y~E?8P(zncN6p!9voCvMlD0Z`U_Q%5WiojS{mSos#7T{8t3 zVHbIz5LcfL!CkA(7%UTE!buGAIN??jxf*l>8je&6b74>|$U6@3^pK98&EEMK; z>Q+pgCRQ)~af47wsUYQ=#r`N7={dYoV-dH`ZBQWKPd!`X4^ub`N>yC>rAJ9%_v z*ukV-SFCpzn7>PwsA1zQAzp1tn3`?e=;hRThj4!jl__Jl93MOH2BhL$Gp5HvJ?Fm1 z`hC8wAHOYBf|Iu?ajgJG9o>@-NB)?9r?iTGQj1-G&F`=!UBY?FJrLA|@fbGWE^2DX z(>}q$xVS3%UVfk!D4%F1xvMM?l1AJfoa97k%otNI+vqo@SmjWy{Y04oX!6;A#o~y$zPpHezvY3NMVyoX@KvFnJ zG}jPBzMKq=ARTeEXmgbs6ILK@6heaxXdU(+5tMM8a4j4CfoFd}_7C!~J4+CS#wkhL zz<-If+%`FhMaM^Uya&L{V@1EJ#Q}dTfM$w%u-t4Ic9(0FLiMni2H+&c(i!*NN zi2HUz+~nFa$5%_5ecMpnk!hCD>1b`QM@a8UzCyI!?#ci+>hK z5~bO$#zz+yFG+pQOI<>;^l{`9?Z$#6g}>#v zYr`i&;UGJ~YFr~CXNBG;=>j%<3QYZY=B(IF&t^oM7>Lh_r1Z%mOEb`+e|FkF?42Ic zvs)0pGVppNzWlGid~b4`rN#bFLzr(w5kebZiIyN20j7;4gVpPVxs`B#W=9ln{LQI2 z{-{@bbqx$>%g=yOILxod>57+$G;5RUHLexloVmJ?cuS}Qo)+e@T)nMQ+-q^uv;2@{ zB}Im+#@_^)@2HQvqwBdMKXl0~6A<&+!pvhrx1jlxy`E8L#{4^hzIaI}?URwkt@W;J z*-ZdMq`}raX0nL*mG2vZa0CY*a<+{Dpi_kRTAS2*? zdd_*$n{hLx*l*?pyv3qWwa0o=q`*|6GC`nJY3D!kj?2cydTk<=KiY z0drwwnk{9(kl;MftCcnJH{!3#*o{QM(jlUNRO4x3)Hq!y6)%YLe5X?$2k9o&w3mxF zbt0qYjO&G21}bBNxa@u4?**V(Db4tjfGiZqTUwRB?-elEHNrKDbM=h44iTm_wiB_$ zqC`$Y68q2O7dmE4i`Vq*YV{IIHah^X_$nx}i3Uc-IV=1F(hWq+z@}-xl;X9p1V)#3 zic(o1SoF&>Vcpk2C|M6m$o2K?c9~!e!S0%oYCnONG?<@mYW&qdf#4p(fmh{fuS2;9 zV-p&~B3=Ckx2{waV&>n%oUu+L`xX$_CFZ0mQVr*Yw<{hGscb7Bgk>h- zWZsU;mB=E-N&HyLhh*=TuM%j=s)`T1M@kE4$Ly{-aiM6nl&mer;$#0;{ESrM&MMli zjbk0V<2XYIjWtFL{NL)C3Mjf?CH^HUiyNz|WEXn$QoB#R&ncrWxL;;z5Em8JumRE) z3bDOVvo;$UGq&tdCQa*3d#fKMGjhiXAunNQk+FROuv0M^pb& zI*W)6kkr`-gIVUT^9tE=B*-O7k^VK1t*{(#=Tg$klY8l>h50)u*a?x>ka5R;Wl99c z=PJX(Tm_;jas<+j6+Q^$V)yBW&9PYJLm=?4M)cFMctjLctatasoy+qKJ2-_SB$^1n zNDZ`8|DO=VGF6LF=6x%eX6f9aTU9?QI`drCg_YPqh--slmJ1ljWlLf17ucgA8NC8% zWX&bW05dGLo4F#OI6t#1_SbXnlN_|SqF;bNG>MLwTGUHK*FK@xIZ}w3#`Z;C#A5F) zWxDEEavZx$h*_3YdVG8Nm5NtcEF|}T2IWr+r9^`$@bA^LON1H&MevIm;L0CGJ{pem zS0;tKFyn12{wmrSCC{3kICEq+;3}Ecn1m+%-%r=q7{|HluWieQdFRY%aNyW$=_&8MlxGWZX;?#`b zC?U-T=l1^-;ym$#vyavEHNdF7mJ(>PW4kp?)0Y((EIuPzQ;zN#q>V=fQCGrUAucrE;93TFrLLohXbM1D<`HDypZqf_Gk8y)JBu#mG}kTMxhuK`B@ZsJ)I62s=|a*|<Ro8n&sLR06P8ef?T9%Mz?>1vh?W^y`706%y4>)3z(#!%*=RC#HOe1tHwjMLs6dhj0tU2HW8TEBj7 zo!Vp*P^RAbJ@L98U3Z0h;Y+J*3gWz_Fs0uT#Y7kdB|KH!t7p|w*JCcVS@AlGV(w0h z4+_aeQ(QoeM5}KB=Gw^CxD^KpV6p>9MW4%v-z$liB&A3v8Sun&EyUlo4fD6>D z%x`E-Dfu1jD6+vSzmFN}NFljBu**(ckzWXNZd%^9($>j-plx*KScHxTX25m@?mbO|vpbD+n00PZscN!Au!$FDC6_2kQe(Inrb zDXsAK<{c=C(*vT)USz@n)!U_GNAjE`|UTj^S6GgGKT!Q7@ z_KT?R3~ILba@|#Gx=~^Nz7(YLa6VFy#iDPdyFY$d(j#*J82=2vo4Z_Z&xG`Q0d9W9 z$|`hnAKRtKTcLK;sV@2~P!=&Ph-FS7B!6vucr?DaD>S*Yvi64JasjjjbGokdS(JZg z(xor61^S%;4R|x=#AAB45OH)zcohxMZwmA`y{CxnK35`Wf@tBd3!!}mSk}ciMDRro zJgfnKB+@M6OfoGWci*jmIEqVAguW%nT;ZGo9pJSBs2GjJwu>&fdyxR6bLNK=1-P&Z z`RX&hlLXR^#8c+D=(0@N8K!Fwexs&}y0nv?EqLj3O$tkdC!x9it$wt+lAeDE)Wph) zmX*SS&qJCWj0{o_yeq^NGaWY05aa?-!70CkMwt*l zA0p6UJ|T!w&tuQO^cQ$GlL+v|DsDo6CQR*j_S?%TJ7;#pH}y;@YbdB%I-8DH^RM$d zJ7a^r`PGsvZ!re`c5E%o1=1bO&@iC8)r#U%`h^w6=wKYEN0+-P z|1&ku>B4e%(ySKemjJmij4r(|Df}_+UsAY)6@<8;SVysMuf*DdGHbanp%N>9sd$Uc z8A_)JA^tj{*-;oF4c!M6uPIJ_J022X5yI0;+0Pa!4aHG!@o7JzJhs7B8_iV-?yfAdg!dpT*p-K*JVfuM&rdaN)b*phmv)1v!21KreSb9|nz7 zd}>atp+|nIFt)c*7M>xR3Uy7N7VB1cc21n+5fZ2#8bHdC z@j)9i4(sScBi*131)S$-uX>h9aSqqwOZ5y_FlojKg&J>O1)x;2zfthRtwd)5t8g_} z>@6S*p^k=0Gb)G-$E)Kpp5JE(K}Ft(--xD+jN`=np?Ac6LM)Fc_PKSCebVVQFn=XQ zVvKVHI1v}B%b2YAza%GeqSUZwz<0Z zg*dO=(pTij29(|5So}f|4a)AYBl`P`-?gpP^7w}XK;lnx3|E4I7$1a&}@L~50D%b_HdmDr-#=FrJ5CXPjOfI5;HbihWObq zKf_n-Br#@+4~{_cg;7SFb@eIywr-tQa^>V(N|`qjHeCiZex1Ar2L! zJ^72BtKOyv=o_@`X_e?+`BaOb{@pwrD zA~3IOcD#Q+&n{GjC}y(#Vif3TgSt?%+0$lmNsQJ*7fZSsN@r$Bomk_$-AlYGrM>3Q z9}45^1g>qwZ6c@~F#_o`{qaKnk)22_R$5SG$W|y>C2uGQ{&Q}pA?G-euDjwnQTGdl zP@Rrh)8ZpXI59S>qpSlzn@Pr$CNt8FBGpsmM1PM6uX$DJu_=3n2ZdXnF#lv>zVVR& zW(I;{NjXlK6UlHB^H~AvR(O|QbQI6Ih{Nn@N8$rVgHQ*RW4tHU5a}F;6o5!VG$M=) zy0{o1jwzD&va!8DWX_MVmrdlALb8Bhy&a!9)-R8Yj6{Ea?&PAqwx0dn9#%}}iF7|; zZB*-(V`BVOU~X%=DQ5+EY#tBhb|ug(8#kOC1X)n%&#X(Cx!X&ktOJN6pIpCET3(lL zEmNlsC7vvEB!(KXuv+n~AXkg)MuF(pLR-<1 zlZy01DC?pwnvT^<>-*Eb28BrV%d+%B5oViiv#d1w<2pgP0@NEne?n!Q3})d;YM3}; zAplvj8bZR)6vfBqP4A3r^*kkK6Sx?^B!JRmZJD-4?~3O8tZTV@=@bZ87K6+N5lTKO z#FfS9U?9ds=Y}QbhU$96l1bRSE-I7MdRQ-`az6D`-l7updo;C3Aid%=Va}O-;ZR&6 z+9jn!V13ec8i0kZk~BNgGn^yPC6ql7?EJY9RYN_l&?5b8FED^EUp zW|1v13nZ)6=o02JibY9_?g`WAO>=}JFl?MqS|5-pT{YZ~Uy6TI7C0T81x3kO8w^BE zM8OhWSHVRN!pu_kJQMiENGLWv+pl4rU5{-G+Zems1hwG(H_tLbB}A+_v61q_Uu*LSs+4r2v05T^mxwAHQ5b&^kjD_v!pRZ^rD{ z_*`&HHpbFpcv@m1RToI&4o)IY$j=oHFIPAp+;vyb>|~DpNnw*z9J(IuUr}uSS)kuh zKoVy2-+vdxT&N7OQjFOb6-g>1%rZl<%=bXdPOd^7(Am}Ug7^{^29>y01O=bqnD7}r zJ3AbhFymY2`%Z~{_DFnE1ivD~tQ+Wu8w;SG+5-E)-Dht|PI}yJ1+>&2sDrv{_LR>~p0>&g_G4oO|ReDZW z?4{?-AhtD{0Q&^NAnXD#U>8A+*<8{tr{td|weP-Igxi!}jHI4*836hA(8XWYbN&(Q z!@gMhhXCZNa6a*45w2>hHouixm77igT@)5>$vqua5+R?#SNn)8OYXzt6fp#tjn|+T(-Uv#LeI6Zq(xUqO&lOfl!)p4w1U^z2=uPTgw650;j) zCD_?=2+4`a*95@-WYUYv3lwFfBd0RoDmv$aLVwDF`whWf)@?*cyVb6Noa#S>9ZXA@ zwM(!vIiz6+>HV!h^MHY&5)YR&&hoW*RkSN9haGnFYhDXt=}s4dvO=M;HxTNkWb7ay zd@BwSB%xz~JVMW|B_^0?$gW=GALn&-#va!d?-D5^d0)>LWFh8MmzV`uaW53+Jerjj z3xg*FxNKYtz<3l-l?Y3Pev|+&3ZW1ar2p9Uex8SU>ad>SazCn|R=n>9C z&5Bu`Fv@_<2SeE|qFg;@NX%P*D8QL_<0e3osl z;zjgx@;cu3=ODEC42*n+^z6@%>Lw-@fR%VppueR%P8m4=@(U0OHLvIpB!AeDzRb7Grc74Hos^#ke^<{+Ws&oEQ+oV!dk zbZ@~qnZP^|`z=#ruVuhte|soFH^ zCPap+1M#Agxd9k&P|Xj;o%a-wO8uTM791-d#qmX{Us5)iR@Qi!_!mCAxWq~qjGeg~Z;pH=yAY$1TE z=(wA^Vnmt$=#J!h^Qqs1P&wjn4aIRHNQ~JXs<{7>*s0kLF9trsn;iQxd{nCpqSw z@CUz`B8$a0MZs2r&BuB97nVy%#2S`DAv@VF1Mfh8s zoAmVeph=3=1m=#$P_0>VEkWuX>{+(dbMES>DB}Rp60mWm;y6PTLo0SEI@G>Sr1_zX zKj}3$3vmf`uGbeg{1HHTcg&7E^qj?4tCL3jpAaXn;`>pJr5^;~18fr&c5yF}REKVj z8d?NJlhRW|#_T^qPf2XL)G2ll#Me6J&W)2E^7DA#KrGaAt~5bTn2BTchk=TN3wX!+ zjqSxJ1oOoq8I0~E!UY~=eq!TveTkr>CQxJCBg9!VVPFM?(b!ZTidMDPBhG`BGCF&) z4SU z!~xOt^OBG}-Jq^a8?LkgdHJ#8Z6Y2KY!l0HLGEZ6pQKAb<2_-PXqFPJ3Yo)Narxur z%j^>*c=rf$1*ESfe#;X8F0M)!A=<^on}QLzC)O84)6K(dchx70pEa~j?2T382)9Ipr&Cmk{QI%2zPug zg58?zI^};ubWdTfESoIN3@me&dm7MuWh=_5+dcy5|MX^t|8qsTxLB4a!#=+0zZ0ZbLH4)Ax z#Vq7TtGHhnLNk*Y`KIT1cF_qF2QQu$;1{$_C15?CBhQ1m?=_n!f<^irosNveR|UX) zMq&<(Qoto4lr#k*V1%3bBD5EXBtj;1yvnlrm%=k2h#JNag&5pjf?Aqbx%E&9X6dG2 zYre$m=B$i?BpW?~U?Tb`ELqZcuH1kqLvHg^+o(%~_~-~h*5X|e&UuvC z0HtwU_7;%)u{>_lgjkKW-ky4kqCWLM2o3a{W;~;3S1qlgQ#9dr?D`IH9s-HuM_+zY zh(E>JBP~T=5>yU#s2b8;4ePuMO!l2~I$}c4a0Ocs%^^pMr0@i|8;`Gvph0x)IyY8G z15V_4f&Pq6Z`DRWEg?n`LWyB?{vN65XJ|7>aG5B-gVKpbA?F~1%mtZDF+XYEU-$n& z6YtpiZITJ(Wp?bD2|4M-kO)^EHJPT3lbr&xU>alQwBfU38QoNd09Lq1`9N2K`>cH4Xw8aIgds zDtE@ZKGxp*|10y;*%YfXaP?0o=tO3)0WI%vh zbu=D7i)@Vtq5fXwU{Wb7n$1@%^GtjEp&@O-X3nuk6)At?2)6% z=75z*;6{`6e+3Jh5kVB6UY`X1f=sheA$-z!Tq`*+m&bQ+S4+b7vdivm}315G?4I9puuhK$v6LUSm8Es|%s(Ow`EnQKHOYxs%2{ zq9yclO?abxnV3s*E&TMV{0yfQkQjM*Qe@6uQyxodA+$4^TTMRISoOmI@U4SsV4G9&->FrJPg{-Gg_sS!x?qE$ z>jNJJ%FIQ}oVvxJFlx)UQ}B$tL{lAPQei}wSieFE++DgzUHbj2^$dhVsWn!H!l8OQ?m1jQ1z6V z(_-Isd8Xo=&T2p2*dB3+h99Ig_q)dw5xG!kDS5bkO9%Wb=&%nYI_upnic{mOb^ zE-yZlGFZG^h`5i7`0w;=zA$C6#bJ)oSpEs%%z5lDS$qocY}7X{2Ntas&$<>^#L;*@tfuovxMZv&6DJ31vztkF$p@z z`lcC|2vz17#Kj&PI-hiYnzr)a6+-no2_d%cMm!^lDYkhtVj86R!%-W z+okxo2y@}+``9J?o2_ns=`BI;UbFq>X!6)Mg+AR8($D>Z|Y>Sh8>HuMfpp; zI%TTGa+`rzj1%?8cTppAjq~dU4^8y;HF_^JXUao|6 zjfD5c`QXpBQ!(VrG#La~4*&~z^rfEIersSuEp^p{B2fN7{`eAw{ebRe(epq&SSQCz zA0H$9|I-iDyf#7k=WY#=hkg##5W}<_-DwFJOO=Yk@Y7Om zMZ!9JX2t00Gw6gI_ee>HON0`-0^~NpbxPX{aVxVmS6r3&T$x*=JNd~|(d|A%I;w?X zV1ImU7YHgqOe_v84-}PqhR!X1BHABj-BiVkZzMMVESNu|I3+UUyHrrFoh_#TpDWWb ziYMpHopvQ1jft&JoRy!)7&dOwbMd~8C@JPGIROFc;)9S&&^L4C#?Fx5n)_MGABlj+An$ zFjBxGs^7C0lpC6!iM5PgJ5eBf?cw>X{5%hf))(|_jJ>EQTk0dS;S zTH^)#K&UimH7jwkD3>?w+u&eaB5bPe1MEi9dj9iY1a+zEnB?kku^9<8d+|JDe;+3) z8J#03WLfgOC_1+|Mk;Kt;uS$IUz@5m<6i%UlWDc010Fb&PNe4Z-fE zltkK%<}NBj1{s3WMN?vPy}~Jey*(1g>V2RjH$?4VS9X%1sd1LfE*I^F(PT6hD;)~J zOhT(Fdgl%z{X54;aNs`zU@j*km3ZG*AY8z7Cz7Rl4WVWMdM%vMV_1lD>FFP9H{)3W zE-XtGro5qO99E=Z35Zen>fJ>)EHpHR9VLXf`Z(Ib+N&QaIbV>Ofei{Fnxz__G@LsY zyEPPij8eBNt=)w<*MOhWXeAfx-{p)Mau~{4LSjuR$W$r_|9ff_5FGfI?p5lkvd z)yCUtlU_)=*o=;jSgdE44iAkcM|hTZG=!JB&qd33RK#JpAudNaYPjKoJmK3KAnCTyZw0^W)U~Be4-G@lN^UAhT9{ zpiW|QcY)pMP7>k~2ktt+-7UhXdagU(f6^BU_ zdmQf9J7;pNp=Xzl4Mro*6K%oN@_@NDO;nS9Mj&@5_ooekCmxRNXjWPP8mP(4Yovrld>G_MdOpf;>k;>v9( z*CquZ;~1nKn+c??IV9?gt3(t_~J1z=OG(_N9(xIhruVi-xGM~;A^HD__@ zT3yfnL|Ppsd(`;?ox^1IRr{%+sohtvwg>it+~!Dd7of?iC=GQ9%LQ=2+SpymT+{AG zbpSy=aYf9dWjwevg%R11+%dV*R8T^XObY((kt&M(s?#|II|$4i4a(W0WD6* zBHRe<(6FH%6F{}+A;iy=?`m@3*L{xWSFxx)iiJ~Ml=B|q5_p#0>m37<8P!i`O*^Cq z1ZI(8XQ(pocq|ZK>73OOV|sQjIt#!U?>HeIDaLynags=j54Y|zpyD)vyNFP_$kg%H z@5@vuyi)(5mvtZKR7B7omnB=j=vl+H|Un}jtGBx0lUc6j)!!8#t3{^ zPkXTqPbff!Km}Fc1Y}&Gdn}!&5+{i^W3}HLj$aALLRcfxgVw$KpIBf%31V*jt}s|U zf-z++t`=qHNR_|>?bY%fRFkGJCB&z$QA|r)g;GC*Y zh}lX2DKbx2ubf<>U@S$_bequ3Hr?rhxmSYVRPQ+EGh=Z|nSW&*w+8}q3gPQ)%Hljd z7k!=h0*k%|KrKgE<27Qbg~iW|kld^|ARtNMtawJxv=5en6S3SW5NxQ&4v&xN(M{D| z8I0>hsu>laa{aGExNazCZ4FS3YlY@+h*v&#c}-((w{E?;NNI9O(ZUVI%K}_=jGzgP zQ;AmuxlwR4N&%El1+kp)J*G!|c;edvoeZ6XG)OC)R^){OFS)mVM3BGF_N*NXMNgHH zs5%-cuMtMo*!0bf+ePH5pfQTM&sf^5-I4Q2vD}zV9uQ5j^Q9)T6Wi?@KzSHuQDLQf zY<)T?Eus6K$Kn=I{`?@3tdNNwoW}}`5t##JmIX^-+7e8tmIUzTGB-NUE|L*xB83NmF>}8Ss`7Qr zjCp#7W%kK^;2bD_37eq0?gMyRki~(WM?X@77tHPFI)~1gbEEe>o?Rxb z;Q@kMoGKy@A8ZQ;Vz~`VTqR1q^M(ixeS#btUxPb98 zD;6U<>C2kU}?l-+7Zqc)g%A%fCGCI7Umzd5z*tS>WDPdO75H8!KzSy}lO5Es=))BDp(D5Y55JfqHVM z??O@b>C>mh{ELbV6+BDV*#Ak8g@9cY8<@0^s(ugDm87u-I4EsL!pvVS+X!jE_I^+a z-3&6_V#XLQ5nk-4ub^VYeW+O3y zwDBZ-&agnL+u79-l^^l!+V~8n8H-AYF^+fey%l6W5NgR$PS(2|jM6!{)cT^FkL>1T z>~p3dOA{05D5JinNB)#Dvuxd8^oW$*Fs z5-+mbmzJ^fC(d5wA^eZ({A)qZOOX)M=DF)yFwdC7-Mw+CNVBCE<$%WTdVGHoVD4B9 zFkEq&6zRVd3?opAvht>hm$?qW`D64zgiprggk_oP!Ad$-zrOf_cYbQxrvGGXvH$i1-VmM^$9S8gj$TJU9NxI{n zDTZD!s2|Kaf@Ibe_n(NhhFvqXFuKgqC$K4bZXvXq=-tL*NLg4_rDo*T7>1W5{3q%c zIyr|qJ@K#zN~OCfP)KTCcq0Ekh~GhU-OBIg5Th2ubiZ_)dk7^9BIk9*10u|rzOe?{ zsy%K4po?~Obwz)Et`B0Rt7j)fA3V?(j|wnLuyJK^UyZNa?sw_-QPlaj7 zn=v=u*0Xwb|2U@4^UR|hX(;v@d}lZ|7ix(?4Udfy>)f;?IYmF@PJ{tY zU9EJEFy~5^EkTO~IDGQq1@<^lb&g)qON>)7VU^j`xwS4MaA z#G}zE$dyVLn_$kyvhjC9VIc9&8JxGc2TDTGF)b$bY*w`2g2%Zq4(-xQbzAVtuLxx(^jjlu^F550S#K#GO0DF(5B73JDU zv5t7h5x)hRYB?+q6pj(^*RusXGaXM8E|ae=66lg(hlO&TLFL2u6$xc~Nej`VrwR0T zIJQqmPKW;v$ebk!PM&jrU!=iaGM0!6`GQR z1iDygwdf~wiNl1^0P{L$#^?V~B|Q(2 zBscuH4{7;&pU^zJqI8U(il(9j?_pAb8Qp&bP*G;ii47j~kFsBZgSdEQ0IdNUV)%KX z0HvNWE1uUg{aOQPno!~+e*%F+(h$bxBKWdm0qO+kP|;IuHi8hTAukYU=HSJ^^)k%7 z*9mi5$lIzNca(|Pi`3Hr=&cVqcj@@$Lvhx_&=jnSS=3W{Mx;?XXbSo8BajFg>w4zd zK2cNSJbF|GO-W2-4KHi3{cXd?5L-l5sg5%#J&&zy+0RFWrV)Z;J&^f*O&9xiCWrjeZEX5+0ojj zuY~d)0`dQ-Iu9sIi{kA|kf0#3Kr+HENRr)|*+dW}=LIAR0{70`$uoDDJ7Mnv5=1}| zR0Jdk0VN2EC@MKi6p##mat;zCDe?O~{q(%=e&@`Y^VV}aR;TLf>gt;X`&&KHY{K7pVZrKnj;GC0Mi>gT_5| zP_UC6(GwHWARZ#j?2@RCp~SI5oC(u#y2Jg9Ae3gk{jt(Zd{P}t_VG=T?r>FICYvI< zC=SOa+Na+u(43YeND-wedP1mk=4HhO8|EiVge^33MNEE~FfK7Z`+O_x6k#qOHpR*A z_W4)mNoq_7jO)Ax;x=Ip$2NI00cMGvdNb4aH{vqEUIMWniJW*N^W>gpo@uk#CDzO?NwnG64_JgsEM5&%y>_@t6i-$F=&nT-T+Y# zK~2Lx^21CvYU#$yM{gG4@h8gb$??HkAkIzB!_r3JBViMK$IV@mw7>Q?Ahp2;vl6pJ zkV`M3A?wwvBArO3Q!i(L?fi~|Fs3JAE4BH%1=&}x#uXx6a84!kaMpb96=~#qV7SE; z=7MNn)UNZ3_;wjkM_s&SH4=LXb2D~ul&PbqPC?EAPtP5kN{+q{;%3x#xg*y90Kmn@ z=et|~G-344Nv$#ZVG$m8nRK$-1u&f3M=P=6|MT{4k<(&`p(S?FtLOdzZ$tcAhpMbfpxUfuy z*#m;8Ytp!&C?APmWkfToF0Au%O&QJ2jP|Jv)IQ3sis4x5Qxc;1EXC{-wh=`ub6pBQ z88I%>oiI<`h#&|I{}Qfs1XloG{fuwdgVm@<8es`ETM4G~ATw8Ddl9gBtSe5^r<(*P z1IQ;Y3drLT_a*6$L>UH64ztW+pBG^mG2J~F1?>0*kQPtJNnxuEEV;xcC z!%sY{*)bf`1yG>mKxUQ*b6NG~Y7>_Ygt_ZsAm8CzzUR#c=xi`6;Cwe;6hhsZa$>#t zi(e7VwPWuV0y|mZ+4$cg3U;cd97{eXlBm6v@%TjF6bE(<#-a;AmK1iE2GhxMXvg zO!hZWWQWtsNV>|ijUcK*_o{L=@oJG2Uzg0{ZV_e@uOzY0iHFMs29l8zFN~DXTzZjI z28(^U$R>qCO58_~M@eIF6zfLOPP&(BAy|$Hfy0$Ka#w!cLQooSQ)WiX!v1dMd&Nch zcJoDF?1UXNeUut^z@>j|{DTb~pE-%P+?9&>&nP~Nyw3X`@vaPAN(}VEuC1gU^;>~uF zP1j!>Tw?}v)ObP!#YVDBx9d0invTFt^eW=a)RZF$ix6a7>*wpgn5FnI+a3tX0xs-oimRvO~I@8Y_w*6;6l| zkQslQ3vg+xwn^HmL|CP&tSc7@QGcxu!iDf?VC2;;(-l)ZJFo8QsM0 z{|;RN+^p!5Z%56MKNRLx;>0Umc{@Rni=fFqz7XvqbE>JMjr&(DA~m}4G={7RVS59PVOEPJTcqKghE2Xy8`pz#G+XYS$H)d zsy2#vT+z3ggZpflT@&DP$$A{ZB6An&j>hUxnVZ^J(tWQNWENp< zoVKXQ(*F?Z;xX&UvPAXFA!1^u;igyX4Wpqf3)nYs%npFF=i{4@~MCKq@x2#~{cw=pc8sf~m_D#NBdQ7glE(j0TUB@x!%5Pbl2r~%NNi66SiM1!tNZi5E+V=P26cNa`IyRH6&bbbOqNezK0G*5#KxIJxtf zp0QFNSXy}Q7O-aYP9aYfx;avd)mt1LrTxsfS%eFLVc48lrLCaZwbf%Y(QX8thz<8tb>pQg zb`)+66NeIG>id@L(+?gM9G_uy<)g<0yC&9q4R&BaolsYKkS5n*V1jZ#)`dG##gJXO zIUKWNl0@4}XbAT%tXT0wL2e}}mn5FX(Zb9UroiseI7fifXRDC)Z~s>&R_T|e7%vo; z5FQhxVCQ)#qPinhYxp=Bj-980&=T!3-Tz)Dc!|3wP7*=0w9T3v@8=(|XKTh1)A_*+ zAF7P?$9bZiDEkuWtge^=;>x1pk@7A+669J?FZMiF%q+qnr=ig9h))EO<1mIP^;yNw zjoL^u-n~SSnL|S+!>hQw42CfeFQX5~A4&*6Hq0`$xJ8gT#w5zDI>LtRE@7UA&h69W}ro~wK zyO3_S-sFO4Z2>NojN!TwYNnKs+>vk%Ikpf)O|e43QP)6xCYn?`Fb}5Ut85OL`KS3= zv;PLdC{j({aN+<_L>TVo@OU8ZDJk+zIiVJ6Qd6$4zK+R(bB{dA(G>2HDsQ&&3(>g%NCE6mZYV?aV(b_*w<$uj%a`a+oww)(BTh)4OxLpxUOdn2;YMw4ZIrGARj}6{S0w>0c zjsoKiAzH!o_*?vMp5Hl6p=hgaU&N@eahI~DMA#&1|F`H4gmELFaO0NSe+8)PC)2}I zcPxHRi$EN_6M(tfjgD11wFPztqLJo~;-X`=2zL!ij#h<(#cgMUxnrrr=Tp?}gyxi( z--hV`^FX+o8`PA$r$Xh9id(m2nR}=(ml<^?$7LUs0OJyE;EKBt!n|TaQ?GbJh?_&= zsM@J7o)+lotbtr{@UA8+df}-%ev?U@=ILV9BH!cJ%qh-^=EhS3TmfGM8IE_#FgT$@ z!dYi`!oVB0RrNSSgzJJ%PCl}maiuUP!chaW!<=~`{4OTL#J?+)=DlgtW0yVnb~*Go zPd#=OKpt}X9mk7MNG|iQv!_GmTSDxPGr%N(9*$F_Hv|y6bxK<-x0gfnBXbm^<1Yf- zY;ut?XzzcY2qib9+QC?AZ-|Md4gu#|qWQ7fL<%`rgbPZ4^X8Um&fMdv> zYP?ti)P?d5?Ng+W>#|xrD%w*wFQMY^R$g^0vD){6-Q38B@&vPi5DFr#Lmad(gxlU8 zj5WuO37lvQT{>lUI!PEJPdm0XvKYi{H3>a&vVLGtp!3AM`K&Qnu=If)E!<2c~t>N{}0{+QU^Rc3sR07b~Dq9pzAbDYUN73DWvJ zi6Xh`jWR(FRM^-K#|;M(m0C)w!Y&ch{EC6}$g&bYBm>0}%$ij~h;!Ox$4#Q#Us$I` z;ttW~NPj&&+xV^!HC#LIpDxx8k&CLsFkG*a=}tmXQ}vHWL!X+gFwAMj?Hy3&IcMEG z_R7xZO=0DMEb=B!AH=UO^n>|@g$*YGxF@ZZ5FBT`0ad7Nr3xkV<6NepTJ#G+@ntVp|_44iFvO!L! zZEUQ_?H!tuam6Q-HcU=W&^Ni)^uqlY zBAkX!=hQ{WZppqPJ6mc=6j)6l@#e}G(~cq>4=;K+_DUjKzfky98N`gB8C&%`1`g#1 zVh<6H!M3u!4?7^X<9$M1F-ft=TkMVR6-c|1D3)sH4??+@P(C>aj$;ZPxf^1TS$?Lqg|CS!&7xw_p zpKCgliB*NstNVCWwWob<98iYg*%$5<)3?qLHZk^je-+PGCDM`H(Z%8i5r#JELr3#Z zeS2x*$TYScb{6z2rirn*py8iq`FW#%iS)Q?uT@9m!Vzd%iu*mh!Y~Svi(K#P!``|c z`wBBB@G8K0N4ku$pHOoGC#GD!D%csYS?VO*#vNJX$;^)FKIj;oC}x-uf#G& zd%_^M-A10=&Bq^61>HzO~tV4rh1vE$|d>?*{Obj?-T;P1~9hy$#lM*KjCE8Wh) z_)wfQPY|7>4aV9>7AZ?Z$u{jDLUNO!w{OOa0?Y=aA|@u9eW4!}fzs{hSXhADZElaQ zEy~zs$r42J&QF5wE|BIQW&N(}8_sbW-i%F;f`BWXcz@(Brzq!IS956R^@ksWxza3^ z_^ja7aY4>EnQJo*^clpCinaCauo&Iw!q=b>GSLM{olj#lu+ zeo43qX$Ug`Sv+weq%T&`4^)<;BP@YM+W)55f5j1 z_(c%y>frzvHzycBjR_+o)Qt6bSA_eG^r<5;`&a(8Pp+g+I0+&*Uw68~cC`>1cO#5; zTTgaSTtsk5Q3NH{{H&`i*NM&v)AvW>DFH4@r=FkI^Dq0J0_OA)5e5+;J4%G1S3~qT zu|$xtBvAfJh?5>1mHzZP0dxY5C=_#_{2G!@pqDXtH~ds6vx+@Xy2z9C)B6ZFi-*w5 zMP0O0ttX4UHBJX`fi>YLkF0l;SQ3>m_K(%iAe<*1lqgMk*ScGfy9n3Ty5;_W9(+%U zA|WNgAF(i@6rO8?oY}^M3S^dyCHEHbj1XAfisSaz^zEsOb7(4KS#iNLfnD|yZW(vP zx&mA#?5&a4hU0EQj#|}y15Tf1$?~aS*Aiz2Y|2p&FLM@{S%Z^~{#Z`5Q%jwd+eYUK za*jOUG1QDp1;F%lbvpiB{#+#+eBR}sr>@_|%D2YZ&HxDqok*>qkx8-D=b?K2PU`8t zsm>lYI0wL`M4QQILY&yP#4v~BXo!Vkm-C8n(lFD@`eKg~OK*)xVREuCxICc@X^;1X zXjI5^$;rQg@L+BPahw8}ljuuTDb_m#emdc)`pNugj$+s_7?%s7fwB74!SS1!#M!O3P#a%Xghv=< zd5P_Xdkt&>we- zbXV-J^mWHeqMZ{Hgs(V1r)I;jS$UQq;T;0`1<674E;uU1!j4;SJV zuBIo~&lBLBdMhwq4z=GXqv7FHcPUo;Q<>0cKbwu%RtOVDFYVeM$BJ@Ubo+!Yb1ZeG z18~%Xfb)VVm!xJ7iL?F;0^?dR;`^??`4uL!Xh_bvPYC?!!B;JbFBPV0CyJNc-Nb9jQHyXI!`<&v)POrg^;S=d1BZ+C6XeH^LXdi|Hij#;9Hpp zwt}2!-(Y=?wRgJ+bfO$1ps$X-g_wimn9la8?;KG^VGeC zK%B`3ZL3^efXU?_Z)Nw#K?2=DkXY(*a~Vjc@641;K%bW&f!e!M51f4sF*P&ln(*Ty zU_cG)0|xOX*Fu_)tVEct?k9j&<@yTK!|S5V0D5$PGfCtNT?c4JbvL^(5voOk9FET@ zXT7T{ekj^i(W3s)vM3!f5u-Ui_;kzb@7IYS6ZwajRZSM}{? zN~#AY9k{u`?Vx5Cw{UfxcO*6}G4@U98VR?VF!Q7jBegicq{q4T9nXq(BQhs3>c8?2 z04g?I(?f9Sux&d*ba0&$v##`uEZB^uP_&EUn_^U8-8;a{GYZW?g_hm*1UfJ5beOjLuNV^Z#taZX$>(A`2-`r!7snsDcbt3!^qn8j~USYs7LXWe&-n}p@A zQ0cC4S{hFaq8iNfc!ZCZe?r1B3TQSU1y(JQM>0T^OxjTEPBVLJirkv-JE|5M!}|PN@>h+z;gj z#rZu?zLhjym^$O&2l(A$GFAaey3kUv$AiEW809Dm`xq7JoOu~ej)CG!4;N5&IvN`Y za+7mK3TY#@7eX5&^h}Et9)WOC2RlE&{e1v3An6NzU^dTJ}XjJyA@NJQHtmX8CU2AP;%sJf16@KGbw z4AjXHp8Gjf9W?AR$c}*N@XUcK5AEiVFD{lH;8x2q;DD(|5m9G1LMZ*t%v$kj#H!tkOQzIYjI zVhTmU8k1fDU|=A&%kjdHNdJzylK01t%)hg!OMtpqBvsm|b!yx*@r^696JG_V;+zO@ z=Kp{w=PZeiX^KmNuYtMJ_?kcykS3CZy6u=PhOnGWy*$C@1-a^;^!fV;(O(E!rk6BfG8@v#DO?20|pAZr+V!;NK<4jM70Z??-GM)KKet1QwE7?IzkjWo{ zxSMjEfr9)A0TX;cpWhMZe^dmK;=Mm^7GO5WG9LelB((W|i%_zsL}w|0gfJ33Oc8py85fUcqN$_C7eu5{ zUY8~3H$QPeO!J#@oCv!AIDLZyK2fe2GrtbJW5rJ$nXWQ8jF0@!9Jy^H=IEBsA>0tW z_*|*R;Q~mF&WX~n@kQ}7Dh6hT(*@8`W?)*%oL_D zlmJLZyL@Tq38v`knyd^MPYHE$bW)yV`5DZeaXjc{lah9CN=uM_r z|I0=4sz}qO(zQj@~E2J2&*c=n!iT z=|XZWz&6u}dCG}Dw&cq#L@u;za`io1L|#17qowIS%PvBRGr;}WGOHJs>amv)l209+j&Pu8 zbA`2h90yDS+#m>b%;pk#o)%~Zb0U%s)ITpHO4r^W8+^qnr-_##|DI@fF(xPc%#zzT zz6$2@v`m>IA_Vys4$NmCfIvW!lz8#Z)mN9gX6b*kDplPQ*)EU2Dyhsqw)0N~(@)2RK ziChZM<^LzdB~Ozv3vqu;`x>CD$ILSpeWG2CdKWwE__+Ww!k}{??q34JtH-fmyY0Ojap?rz2x8DNU_ zHl%BrB8f(24(f8z&Xt~qIuS|wxMjgSM)2S@;dbUxp^`-qkWLSO_cDpTr*Fz=_x$m> zNVv&HVrrbSoWsq;3F&A0&fTg$$Z9i8ffMNLb$FW8&tH@Qb;2o8xE6;k?_}9n^u#$L z+=Lz2Yc}H&0l5}vY0|dm*sp_8-WIgD(^lZyT@2kvZ3g%wKG_0QDt1Zt6*XT(Pg?%B)5!;*Ta3Z!^1<1K`q{*f57+VcQ z9GTV<_vzbhj3al{0QesmiNV#&IFjGF1pEyUcUJiaOUel8y4L`9elpHd*?R@KK9rKX zX<&1NxeT~tl*Z{KA@0KRm=q6|389EfuM<8Y$VJOeo|M$vWkC7~Be_2oT9be-8&}*L zynmT2z5glLWkcHNV*ZSG1i8b|49q=klK@-{{7Uu3Hzz~8Y0G0vOYG~c#ZRt-OdW9r zdA5+;*z%c{jHs?!8_-$spd~xSe+bCy6kTL^NW#=`Ec?v@W)q)!o`li_=R%{Wkz~EV zI)uy_qg7#HT0@8f4IwXxExQYHF4zcT_~ka&a(D73FrSRAoiL zH1R)SPIv&X71EG?XFVWKgyWda{841C9t#Q95bFbwDB3EHtq$4%D&;qQM%*pJ4Jcm# zIP{YglTdT5zn6P{GIKQO!mtU%1dg3r3g{|fMS>Gx+Lkijr&JAVy6Z>u5CfL}7O=Ub zXCW~JJwk{pLyeP}{VxT%^i>{t^u|1B}UMb(+n1=&1V66HsYCfP1m|>IS}K*-CKq@#W+T!<8peO zMX#Rx-frvM^cNZd1W?K?inKO^(jW95yCpCzd>aZD3Je@jObahE7fq_d{QEBba_ zBp|XS6+mHZRvDj(ba#}q(!8{uzA>oVOYU7z-Ng4dDH5t;bIjAr0^B>%R3V2kDqr6e zjGU*nOpU9*UHr~PQCo(6Tw-%;s3xS%O;~PxX|*U=Ef(C2$Od~^#DLR{6hQ1(yT)Q= z{q8PU>tLw-Jk#tlHR;5kMZ4NqKyZ2Q9RVI7n02Myum;F7-vM^@Sr`)gK&J?G>P>vF z##GU+s*b05W)QXCcfnve4ry`l_-9dhSaRsto_B03Zvjj(^tg`H=Fu&{;h10OZ`R|k zz9n$xCG$(V_nENV25d3th_OQ%jEeT4cIc143(M@Ip?nYK8iDRWJPX6R6nBK#4o zYlOIupdB2ImqoiL(XnKxq$%O`GSploub8qup(gBz3O83|Fjqgg8Iz3GBOiJR^n_Bp z$E@`*;m(VF8pjgp+TT1$pbDXL$z)<45CfmVF6>;o;$US)*)^@|%x>8oip=SeH4bZ# z0e>&hvsQbpJ#NV)+az5mUVcY@)%4Ac+1dJbXV7C~>ijVx%elRpi!&3%-l5W7C^FK zFs9R^O_hSnD1#aqRmGMi#x@DtCbo0i33F^+h~+^=nZoWa)GT5Hg9;6$bP2KACGGxU zf~W^h&MNWK@^|GXne@*kB>Qd5y5j~xx%2iUhxS9e7Fpq4P?kn`(!EJNe6}`P1zqq+4$w+p$??Icv!|7pqvEn*`6rB^rsj=Jc5X_%G zhPhP~b)UF<&jp|P_H+Op&T()nDr>;gO|%Bb#!{H3A120~-U~?k#jhy1M~+^`!nq5> z5f#{)!d*)?)~e-tLh|H-G6(M~0^AH}aXFzJj&=7c!k|@QB7R1IGu6#VoT+O+`GdVd zQ~hSlh;|XV2V>Mq%RIUd2sQ5In1X)#eFsJw9*fB$T!=CDU_CKK0EIw-8aiAa_^P{!GO)Dv^lc#%?SVokZY*QTyP&KfN7@PFLh&?-A0EE529~=$<3iS} z-1X@!l82T2#dShR9<8tk_4^q>HKEC|9{Y3^iC5{l9Ef7rCCo#B=Wv(R_a*%#- zuGsppnU*7*c|h*VF^F)IX3)cQ)`s}DnWH2p_Rq-%$97q#2{%VNx$28R_LdNOUq4Q2 zKhd|j!>*Md&+9INGZ(T0iuHSpVC%-zh;}i~>~#VyjE#%*?F7I0Cv9~ zuE`5al7td!4sy*Jt(@kH^%OJ@oe9q{ZNl8^^fV|=eq!S?MMkSU!)eQ~qfiFBERE4U z)}b_ZdAaWzeN)%|9yIhf4?#F%NrSfddQhO(!b!L~{LVp;ZVV1Obq=vzi44i-Mn`;K z$V6w9-j@Y@Oqh8jbFO+EDj;Xp#9kR&sDZdlpoX+=R9#y}>10pSANLINtJ{L}jz)YS z+8suwIJoz=RoBPDoiv*0I(I`TR07Q`HqNY9{jqF=aIht`FXO#e z-)^K%Zu_y~YJDCSYCh<$Jwvb|#JRF?Nbo1u1UgqKbtSB9J>qkH+gFa(-zmhT`jB zAeu2XYrGXfjH ze`zYcShx#k0}&JzWl%YsFOJ?nDwE}a4;dAacBR9J?#_j;bu>eh1UNlT!scRV9yLKZ zYi^`6DN!Rt%1edAqcM5g)Hj`sRW-eQz1`tOGOXb!%sQf1ko)0)c4Uhl0qyyYIaq>F zQVlO9m|!^SOTJ+*$h402D;R*kA=*8MiIx0f*B^nn4qYfd2jfWrn!ILBjjtWWw_6vM z@rsh>)R7XjK0=HBKLzGw@YKLfydML&z_Lq-ABc9N_oD7OMx;Az=?)++Jlc_1XV~yI z;tA0%uHRdXE01xAv=3*7={OeJCG<{R8tB&qI+~_g8fL?vfVg-p1NcQ=^ruCT>Tr6E zWV(>tQIj$fx1%M*_~FiDJ$4r4a-$od+Z_9$#X$|bSS=eTmdvk8>5|3F9Eh1 z>@?>886jYEFUP4nT51$`OMWfXjtwY__$=)<5Ul%wy{0=u(F|YyF}CWy8Y43Be@QG?OpZt5 zIMH;U!~~2aL^20W)A95zC;B(!BkCE?iF6s#9L!p>%Sk{k0}o-T{ZA3%K)M>CLM(W) zlR+x$8|{moM7mA!`b&?VEr23*k2Z7^^jKjkzC4+*&x=8mpppqGVe8sVL*SMzu4)LHSqzRgz7a;xci(Aj}M~$zzS4vFlkyR%tEP!?uf@ z4d(I;SL)oOxk>}F91>Zk{V$+E)i)ipwDAne?kB)%=^Ce zcuj<(V+$8c*L2aNL&*g)mrIykmd%K!Q;BNf<3N4*97+* zv?TG)7v_==(G-$aZW82zjbR?$A3I)D#KRSQ9~bC4Vq6%x%XS2R8W%&kv<-}~(2CuW z5jhG-?~cW@LR=&{SYXe?1hDWWfSCzc)nm>mz~$+a&l$QozJE&$)Pa2Z^WU$cN7Gxfw)72IXrbb2Uj0vn#M^- zj4NHn?=CFXaSV=cWPtR$>m68_%oIQ?FrSTfcSgTR4^xcD*+iwq{8*u`B1}NQuf$IT zW!|tPSb6+AfmEkUs(^b$n8S=i`f@x{CPPOa6F1)%3M&JC_5v-=eU z(RMmB-~l92PI;)&WKoLEg)si`9};`&+jUNFipJ3bsA89Pk~n-837eA>dHlCy`r}JM zm*@xQtMe$PnfS9H_)BrqqbXmAcCFc5utaF9-S~q8vG>-q74fiW^AI&>dax16bl*P~ zQMF6bwVlDJ3xr{8t8g_T!Uad2LGQRyfD;+ zfBU2`wS{aozx5{wSb$nctM|V$iJ7C`1A0j0#6-${1szc_`q#M<*p<}*3}zZA^y>mS zZ&|M;PSvLi!i=uHVf?F%!fT_L@~d>umEl-+SlowWn?DoP1(woK&ydCUgt_{703zOx z1W;%xk9i8=((-pk6Ye?h65#yxz_=E#@B9Ubeqy`CQBkf}x@^GSR#Ngqf}K%KRvxkL zRgNW3r(>~+sN8S8*6k&PjJi0mzzGCK7jcRpctgcid_Ny@xj-k3I5#{L2_T|8w##ap z%xmn#g-&w-dzSsKgK%O=CrmFtSrEU=Z+YA-0&Zac#vPtX*F(81gNS^U zI9PytAlnxiT`-GoaYF%u$m9*SCeTeifbfI)9H&<(+RLzD>46cayTuxrJ}nM{k&7@!Szt3Uj~K)Q&{B(5)cmw?wjG9!~;rTlS2$)&36M+e>?ZwjEeOkF$-9sd<+e#n`Qrn>uM zP3b-{lXa|qH<%|~{EXvBf%OG*4WYC=!UJ;8H|{BNlM6gL`jP(x$$deGi44kf1(|u= z45P75xevf)?qlN>qxVBQcaA7g1c5F6K#>kRoOEAi)e>SFW^DJz3_(u1(qBcfd5!>g ze;J@n+ywYn8Loen#|h)%66nB0w5)CWJr*D2 z9|w8Lf%z=ndeA>s^|0gt4?(y-a_5I*-exTFFqrv*8?E#}_3?s8kduH~eEAUw4Tjdq zF`{o*B5gi+s7rJD5|0*1pk82!?TB3k<&H~EmFO1WtmXJG*?b%-%vIweh+aKCqC_g( z(?V|lX_*kly-aQw2yhS8B?PqTbn&Gg19T;N>$nJ`|E>FYkzs>}JF%qdh`WVpFk;77 zdxCHJOGlmSCh>(R*NiDr+OKpr^pz(ap>@ivSXtjhK-bRxdp}VV<4m(gHTp}4$wgf( zZp`spFd^Fc|D4t|p8Gsi-=sCH7kA>Hq8NjnY?_$r-WTZ_R{H14evFhp73cyMiWZLB z66|7dnKL~LnjpGIIwpoUpO?w$_<>~(@C$*Chm8l@;^A2EsWMx=Z(Eg#?!p2cuDdqG zj6E1jln~N>8UQy6%JTu$W@vpR#LbP1B*b|}*CS7Zd313@4u=%a6lv3I=cf2+54|7h&c#BNki2C0{R6V=Q6Y$<0nd6uvT-edKi_==|8h^3(U<@K5M0`r^kT za(BzF2A6yjh>CGqr`r>;n`pA*Pze&o{xAn%f7D&+#`ak>{R6s-_sjkx(iLH)8gAkO zoI?NmEl{^7F8b%uyrv&qo8J1|T4%hakREvGX?PqD*D?&Cw*{LM1C?Ga1nu!3VJ`gu z*QiDt?eX6-m@Y%5r$iQiyNHTWla`Q53<=94i1AJdf0jda;Iv&AnWd^aTbMH#uZ$!m z{BO&M@`lYB!j10~Nug@c;{2EpxZkCF1pj&01oJ>WrvC?m39%h_pkw+b6dv(wvBG;f ziZ&ygh%gHqxIkn6(c#BuLd~f@67Gwo-v^+j^u`_&VEkCL18`)+)rflqI2}3ZNirKg z!i2g%ewR7fh{y33tdAa?7-9Im5aEl0`>|OzsdivBE@qxZ` zTQmo`jGmT^$^Qd&CJk9s$8sM7xDwNN}D9+A$zQ31H03 zA7K)}Ii#Vyd{4Z+N5~IzF{tifrN&IW*nH%|+{om0Po@Aw=&v{B zt$WdhiEJjK$K()J`RmY;9TSh|6IcBe9UN_$oK&eMen-vR-nBX?cIFH!J8NzZkaehz6 zHwvMfXjvT{?vF`J79p63wN1WFi1SIdeH7--rGQ*Qj_I|*C)~nI1Clux9qTbgggcOo zBeR7EySlF|!w()I+FLLkF}`>0Aqx@D@I6hJV&WqNCzrEgb>;n1!NEGVc1I!m4(r#|b;tDO9l96Grt|gf*5LRcX{utquY);HY=*EHRzokp zLJ^R8Mg|;IM3~Et)f6cqzBDyEv5>|>=@xjD^LCaxX@g}PAklN zE6)S9r+w-7R{^4xxn)l3BXN{yxj~*2E3C?=Q>007>bsTzidB<4zaNO89{HXIv+P={ zfjGlntkBfL>j-k$*`FX-=n!phf#!hr7VI_l6=E3yeGF0zBAn#efik)zEEY*Pf1yBU zgY6c_aD#EPAUaYT4>&CS4ZfWvvviXCy5h7F#Hg3O0;jTZuRyBKz9jhpS!fMts)xOH zPmGFiA=$u-<-c9CNVvU!v});OgGqoAvXgPk2cq1`&~)PCh*f>&WU$N?-Gk*cKS7x* zT=-$*z;H{T<6_du#*p1X!aP_xzd{T{3X$2Zl5|kGFSn4DI7XPui<4D3?7N}Nn0K+@ zYe(EB$Qfhrh(1j9>Rua=YIh9t>fKMjX@boS8cGpr;t8Xh_>B;%+m@WYZoUqbtJaz3 zQH;+85nyP9*7Zgfz7Df(k4L9TVz=v+2nv8xd0f%3~Y%Xbq-_vpq0^7-ZOHt?qVT_VgKofU&E z^W`Pbm!1N6rUbai#kTf;0u=C#Fvt{a(>7Jq@O1B^nz5 zQTpWPT)L__>RUx1Zr-IW)OR-of}^NoF}2)5l#9ws&an6%0px>(qZijd4O42aSV$Eq+lij+U{K*6a@8IXH7%qOfS2Mcqh5CQ3( z!*Qq}7YNtOmEL;XB?L~$91=M-UJy-(V@1RZXbXfZgn28&dtU)rgp=VweK4K8{YkKM zl9w2*4quc}c>}YPIlh%JwDYu<$?>7S;eyp7aY`E`d2+jhCm5zdxM{f4H5hM-mT*&J z*%^%arkj*$G)jiuMY=-F15#)nk`XDZQCEWeFh^ROiP$sUwPNw@~d_v-qYeC|dQ{ z!hc1YnTS@@Y{>cty7Dw4>gscrF+NQdz@i>Wl8*aVQQX3m%r zb2jChmaFvhetDBKuahz`h&^ah53TWShagM$5Zmo}|PID@*P3`dW^ zxK_1FpqIH(uDQ0eO-*KK3HM}TCkkm^zOqb}8$xMH|GOY}T&^z+#?_)7v0C$i9OL2| zq2^2#UsP=7uPc+{=vUn!t{0S>m#rq(R*+3@6bMJCZIyQhUKh!rfr%7;%OceVF4gJ3 zD{T#8p6baj&cqsVvoPn4bc3-#th5b?%Z&255}S(7A#j^TdABP=3{<$zx4&qo+eHXm zvG>PW!f0Due#QRV@@+O^_9;E#;DF@OA>p}4jOYSgTqVSL(+Yhtc{^y=ZwSv*dy8~N z43{RY&O?bM=JOM7G95#$o$5Qg``${=!YkGL0$&L)U+;($p~!(&2nzraIB4e_@l-V^A4*4JcY z%thZYXw-#5Uh9@9yTC3*dOXg6BSh{3ftbgfHQ99RRAEeC+VjkfS424(?i|R7JMP{U z$cq9u6&SuB3xHLlb4TOK-S~!8oSPx@e-8pKaD*f0mD6`GB6i}UzZTmFFb}vYfF(gm zeq@~71<{qW1&dhCVejmuZU=Jtq{H+icXtc?0BVBCXm)jgaQGDD5um0-HXh%E@tvIV+a4SWfua_!s>mPd^=qsw#@ zG*{TC#5FbO%o4doOeQDGcLFj-Td4gC`ZlX|`I|PrUWoaDI}@g~*UKQ1IvHRe+8`{O^dd~=!7aj3SVJ;r8*3j^b*B9;V=%+!2y%+JmFwmyUvDxrS#0?cKoiPDWFbB3}*^=EqT&`*U#hH5|Phd zmfi(=in1`5vsc&CQA-OnGrAD3`(t$hW;oZkDzU$43d`xd_Czwi|LSZP*%d(4*Uq9dWQ=Cxn|Bgu%E#h}LC(U=!D`Qh^Segen3G_#kv>go4rZK2+$P*v zle6xX&kI#+&2+!;PKC;YjcXP0Kp6}3u3oYA89{EMs;};?Q3o+ohI-gY$I?Ru!0`=N zZ+c2#pw`pd6Hf|YRzrD-BEIz?hsR}-U%TB>fa@!x*F&ASeE6A-gjy!_cpY%3I31;?~f%n7n^zGgFr)^frpnyQf6 zV28%UHUgXr`%R`ShTt!9AO#IV%L@<^cI(8xyy^DV6AP_j2> zid|k=+*r8lfe6YRMfC3qa|0tkup`k0>zxI<@Y*JDsq7XZ6caZfsq?IO2sD*Jmo;4H z_BwUJPP9!Zp*R9UQC>6cbR>Q<+is%}8PRdva6BNy9fu+@d2D$ofJ>S5hw7`p5av>H z+KGHCaK#@2lAEmKw65$b+9{!gM~pj1fHOn%lMwcBjzX*9o_{Fb6m8~l$e7mIBt&XSZbR%JIWvqF0qg6YPJ&!0*WCpy8 zBpaB7k`<>>EitYj6YW@)<~fclV=+&$dBujKBhD*Py;{qOrA}kkIsM9nmV%T()_FCgY6-rb6=Y$G8afPTu)FP2XmB z6;Cw7ah3qH8xdIQl52%HOU#Viszj>EbUBv$F=5;oGCE2-=5>X6aB2G#w~40s?d|RHiN2kt{GZbDUp?BPCrs#f z6Ak~c@{oq;L6NR6j?i(|`I-P%1)DKW#l)`Tj{$W1%VSt&DA`0L)QsSIPI@2s1O+ri zYwIPixK)sgLT>JMCpiSMK}ZwSn9)7jf}koPGpDLfl<{Z zC|4J6Y^W2CEMsEc#;GX^l(@Bw$&nI^Vtd?^W9GX!G}l5uaRG3k*dL3FATMNIYLS#k z9X|!ioWhc&Cx%Oag8~c!-YNlj$)@fT0Td525^ZCY$#Eb|^l0P8aTO}ct3V$+^Sgcq z2@CquG5?ttSXH&I$3KLcArdGwjP4TTr6)bh3Ey-n{oXQuYCko7 ze=K|gfn6gcaDdGPOvsnaW0?1*jP@5!50kgOrGCkGDkSd(|5Zfh36te;+$6vS&09CQ zT)0)J1{mf8|EF*FUWCx0IOxP837jTVtA_-T0YkYfe)%g1rzgiKGI^l`&3{sX%D7N^ zc{XPY2zA|c?IuY-B=Kdz=2Anw4XAG(DDO1>Dk%3jnvPYHx9T1c=;~GaRJM0d2B2*C zsThuDe+?lyrVAsZSpHNfmz2|c^vj8J-9iZ?FPl8mBnm#X%xaC*PUD-T&=_&WO9UN- zTMaCLa%Z-ikcm~T!PB6O!tb75VA{az%xPXs{#Vl_p)D0I*SH7kb0ok7xq8e+TxFzy zv27U+AvgKi_(_=s$|G^B62B2fsi#kw8t?1d9HGacd06m_A}`i8ytfU;qJkWz%CR*^ zy(NH=7i|2hfV>#u7A{$kE-B0<)UkwR!Cwe;XPGNIy8o1Phlc5@XYxBmlgIbJ=^Iuf z9*xY6+eA4g4?E(}LNtZafM#{UDj`R0;nnlc^zHKG(?Q(up5^3NRM(sp7mD&kqsKmJ z{YB3PF_YPyr)upgh~Yd?o1=U0Tb=`HJ`5lg@IF8ZaLI$=v&^}UKpxfZq1Z_z5p?e$ z+36iAfIh;-@Ui&Hc@Pu<-v#Irq{f{g#2KJu;<%{)0stzVPUn~nR=&_tNS~!p^TQ@W zGMCwT`Q5>51(Hple2rcw!u=R033Q&c>AUT>psp?Vu<89c!xLntq2wReEE$i?gW;D) z$IsFH9pPvro?3}Z^zFpCF)d|}d`Vp=)QRib2*PN*SO(N)DW6GfaS;LC3XFO#)x`IO z(C7>XP8n9YxcHsLQrESAF2osD)8-TlGESz1sz=Y59JgHJ6r?92Ur{a!jhKYOl`eJQ zHau}G{d>OQRI+>AUj*D9$K*+~%Hg8T5abSSpm1`z@?{0aMBHDO1$MVEnl0^r5O?EB z0Tic=UVMYTT>uu)EE%GV)=1&Vt?feGxtqL685fJ5n73(XPB4-f{uklyAaiT@ zs*Gm@z%HrdhvP?ofFj?%N^d;;N4|3d>w-CQ?k}$ZQf1K+#iJt3NCY+v%#wcQdsh~* zYWPypgfI`pp#s~ViXm4(7Ngv8+e7hO0q&>Ijl)D!F8UDWSS$S*igBbH*JvHr6zN)U zI#6ev2*^E;>7!G6woYM`Uzc`ST(!hE1UcJ243wBaA1;w@`6}mn$X^KKg76F{9^yos z4RXVyW?knhAh^`pIx|kzw}aE?2V%p&7Qtl)ErIGMg5VlkNY2MZc(~CdO=;+wG5R+^ zuTz{F6o*tV2sVdNL!lH;=8wnyy+GLs(j$?wF%A~$B=C!X?=w57`+snr}qG_dMB+!g= zu7#W^emW`F?&YTIK(XRDs573@CvBV*vhji_Qe!?&^}OkNXx9j<4qg>NRYv?bZYW@w zlDrqGO%j|ZY;GwcUt!#+pcbYE2YTX1H$u8bgEedn;xYlQQKQ0q*%M3M1ma<%>7uyZ z`6uCSWqD(Y({6!Q517>!m+CvWC=vli$|wwPxfRe!OIJ-V_~vZ|(!kd=UEz-5HZy@|?*70mqa-v&LXd-yb)Y5Er(RhJLv_iQ&d( zG14xWp%UK`N;=c0w!{f{@ogSXosR3|3q?~Be1m9Xt46#pL#$46rl*x);kyY%AL&RF z><*$_dYnC^1BG1$IawTq*5e_1~rahh$?P82_JJ2fduRj22R^!%J!jfd&4TVzgPF$7Tpl`-lZ&y67Pxy!O zhEvCx|AO+EQUf7_rtA363wAcRiICv{-QvX(%y%YW!Ak;N-HysAra*)7h9GzK_PXAS ziVuY_?sOPSs~++oq+1BvTFz8T8XJe<`07La?%dfb=YzOu0?8=PEfP*I6XbMIkb`FjEJX5 zW3)$)5KkF;LAMe|32+%OdB@mBwEK?rPv~$z73TI4lat?vM$CE~)U{-^%aH)nL;@Y1 z`KH3-MstnjJh@EL9UCd81JO4JlF|Pr!eH?@)^C_yb&7PeG!V};yIdp4r9coH!XFds zQLOv~s4I;@U!C>s8X=CFRzT#lTgpI>Yn1lyb>}Y=q!lg@RfHqk>Tc}!8-i)smp1(JInuF&D^Oo83>Fz#OeU zh!1gGj|+0657p46#Q6f;P-xy|y>!HLAaJ;kn-J}BwW!1@jvc=9JcKKUf{t@0?uO9V zHNj3)cJQ?r6_C3La;I8$(HEUgJ9h=+ED`SAIF#;=`Cfun6R?rvL4fT=QYI$&POh3E zKD{j@vrqH2=CF5iBqs5}&h}XK!%cQ@0ujz;WL2J1q{ z&sun(5Necn^TTn!05>+Gl%5ArrT!;$q8_}?_s%QOE+#dx5LbT{j4spKIwOvr_-?_t zNZ*djZn_>nd<`15S7@ewt~wRJ4x!$Rk3?q{qk`w|&FcVe3i+)=sglkc?iQT;fu1)? z=5Y6wpk7fONjs5s-Y8NWulHjsaj5`Wy1R$dw?Acq1I2pWl8IR}Sf-yBk#nHjx*E#6 z;co(FMx)Eai&;V(46ltkzoQ@TtYGFMSM`{PlJ;cp5@njkpK=m@zxHp!<{1#H$ksafKQ&THh*yrPAO7MTnaUQMijUP2IEtqY1dd zijB>3Z=2!-lyic~qMWSW8NtahJC4l+I(HfEvLab{7}w!F(BlgEV9G`v1ki+dEvgG=^U5?=l(NYm1+miW9(1C!-J zJWDVwQ;^1cML~F02`A+gC7J?VTy7L{v+523j6|LM(Crp^ACipGsJF#oBFr)bDH+sr zY*S)Q86q(_V(tfo%Wc9A4)#AQgtla5(^*?A@FBFTz#5o@!_@@2BvPzl*D8Rlxw}>8 zY7>MTpZ1fTwvrhU>WpwQq>1H6LNqI~-JoL zuWum49A%@z-hKa10bDWmXz7w8FI|0B#6uR5PT(yeo>_FDfIHPR9-ChPn^m0uyI4mG z%bgq0ijSWN%&o#v3Kd-HOS>eN>rKV@g9!TS4E!%&sqYDHRk~$u_=O!7Z}dFJM|PwoGr|`Ve5c08~slSC69QBqjw8( z0fwYQ?1&W?DC5z8ISG$TONiruWVE~Yf`powpJbAbexM^pz6_c>7}AMOZsQVROfwv^ zvjzI6D92+C(HXmpl%Fr-F|5*Uig-m>9yPKxhg+|f!FZZ?6p;=02MZO!*y6HG55)!x z1Gz~jZhlDCXcMd@hr=BnXB1_&Vn5UwM`apq076|Y9v5xKj5L$qui-^N=-Jr6;0P$@ zUldZa6fVb1xLWBr0DiM8x+4eS7@-pHh%j%kDw3$m(ehWo+@ri%(K!EsK+0JkZ*c4M zZc*+J9I-IxwtN+U9#uz9p0XI<&X$F#o?p+{NvLzuEtmA>bHcox)n!ns(D1{^M<}r8 zyw+HLaR@U7!?M0OOtdp+a3C(;EWk;RaB!PEHNGIs4TZDo^<^&fm9GI6dNO9$WFk?E z>B1eI>k_&P!d7&NBCLFtaueMbfnOJ%c{A*1@P^~kB>|mRlKYqhZxS>yK493BqKT*d zGTRL=x}Ri_S!tA852NyFAJdj1B%Ew(X^C_74bz!TP;sojG}J^}4KmqbI&iJ9iMGN{ zBOOg7(7mLARt~HGgt&xs*$(;Dct|mGPaf@0CwluYQ)EC#&|!La^a@iE>O3nF?}_p} z+FPr}0?Ycxsp%fZCZe2e^4G^`7vkX6@y@t7Ae|-4YTDbpxm*!v>eSRq?+M9Wiz%`% zzO}p);o2>Vk~mVNYs{+=G6$hgufIY8rkhboy4p~nM;v`iI!iioBfc$oqL`@jar7mG zI&y@9%yC8jUG6NkIqrx*2{Ge)B*JSv+$YS5vwYT(Pk0ieBc`qhoM#fYr_8*vZQ4et zD=yc3GRiqwkkgbblYBFtEsS2TTP$&783!xvbbR=%Aa^RRl2_ySm7vWkR4Dk`zd?X` z!;X*VSTxnmS{abqG5X==?V{*jv$$RSy1sKiv0u?0Rsl(~2@~U394#{UIGyKe8*rj9 z_0OrTvG%Hbdql}VHC?usNOOzuG<>Jcj6GI^aH%W0oOg_9v#3WZCONAdiF1WIE5s#) zD$07Eu-t3uf6}jBA;>u^gBQHvcngrHS)BB-Y}ju!3y%B@`TH|CrwvT9cOWgWvlD7lC zM}+4doR69pZz~|_0o8OH333_Pav;;QKS?dOm41L5J$)=8JBrB5UUQhUy4?i026!&< zv~-g&&$yFVK>kt2VTMPi^?(2m#;G$<%)Tkw1!V7*o{jlXkhv^pBt4z+<7q`89Cjr; zrPl;G9r%triJ8zY97hry($AU&KtV8Blt{U=$lP=o$S`Vl5#m9h-KM%L;?OZdJ+1ce z#2y2=gk4qkB67Xg;h=69+i7qG%Aiwafp0X>KjFyiSf%RHaRC;$7{4qt$7z-h8+3!| zDxqeE-0SetP~1}nOnMgVCs>1?5bE0C%4m$8-E(DBt{`xPHXLsYa{2VGFcT`KE&Fd& z#Dvx{?!iJ_Vx&yj!pbCW;f+CEV)=!t#xw!BY&yOp%nO1v5_!mJ!A<#giL`ZNR3#+M z!H7Ov>t}ZhMpm36lHYiyyC*jOHiT1_t2O*p#iE-5F=%actTo^6h+A=*(*@kb|5GO-udw#Gj0FBy> z@;WXM;gWMnzB^tP?G|ETkpR2yzSb)y8AP?ID$k zY}C)p1Uot2O%_28T0fiU!JF~l9YDllbj|hHbVn$cSK&M2-vV3!j(WT5@<_YnPGD{a ztSq$L$&b8Y_aNQ_%WQ>E+x>lMyA4jijh1&u(i!@371u5 z*ISoGxtJHL3(BQq8BfKSD~ueG8<^6L6lKm#LYUfqXGl0B|4H%X*+pzA%F-5ghOpcO zddaIjb}U25hy=@r!Prfh*@TB(c4!xu(RgNFM^jG-AtyvYY+^)E3ZzwCFSu~mB9nd$ zAQ5NR-OT}55V^t{+lX}f7z9`GxKl<{z9wzqY?*b48hgkAaiK^MFv*rhSz~uddVEXU|J+yzFBG7RT)c z0dF`GVl*&GEWEb?cvt0djL|q3`HZr0w22ZT5q46vE_8HrB| zqnmak()7f;WgLdR*5dws`JFG2i}< z!#I|p5~ql!xN;rK{Ku-zn#E|oOSmHFO>6b0n_J2+h{dgB;BMe=8AQk{iG|;=@AB zcof;(Zc1AE11nC8n6#N@sf&cTkTUb(CRj5b5~fZ$b#m-b<=X{jHLpZXwCAC8NQhih zBVzZi0?`d! zim{InmsLK<2Bf*_6$VdqJn?gVn z3+kbpl*m96b!kV`gg74Bdo;>~Y6x>%=uIjOou8Eu`)i=V*t||CR|9n*ClblrtuB=I zo{q1}mV=74k;>R+P^Q_!VAqID5nd88(Z^=Smxdj;tz~k2Pv886@=^W)cWXd;0g*tJ zJUJ(W%|GdfyuQ1cOJi_95MLLOJ4^Bgjp#ERs|t1J!?~HfUw6eUVNMA>$(W$M1)0x?ygCX> zBIw70-5>~xdj4jgLqN2SuoJ|klPIU2?B=ACxk^}0ol{TkvR@M9nFNuXzV?m)v!Y+G zhhxCkh}xk=C{{6+mzM=lBW54@)6UZQ+J}KUeNa~D!Pr3%Y|9=l;*X*!Bq!mRUL1co zlyhg(&!Rbe1b|D;ks|3bW1KC_sY)LsC%)&F!4M;CclmmWr5C_t>N@#IqI!tQc}TlR zr!J#f?v&!2r^M1j(_;BrKPqz6qa<<$IsGUgH$P1}5~qtc`(dQCWvd=tL`wHUNoPQi zXEk>GGC#31gx7@AcV@Lti9L@g5|WQGTIpgT?kx=e9*p)ifHyuC)a@dtVYn~ilGP>x z9aMa&4Rfr_wf-_FuEXgL;%oxVMXp}bZfrgj&ONK}+loMzV2709g{J*S6w-X`|3u#tlCu;N)``^OH2vVBPnh1S;hqv)Fo|~|q)T{4V6Kn$qSWVw65>83 z4!GLmRY5MtU=J_e#>zhj$Spi7*L`tziJ(~0t;P?8IBA^?D7U?T0p!Zd%~t$IbZ#OH zmGUi$M}Ya&(}Dj42o#T z8k96%33Z`)PDOn+hjiU^RZ7>oPF7?O1SuPF@@DMdQ+KisHt%KV`)bA~tR8A0p*+6f~D_V$cMkI39_`zz!0 zio=Asx3O{K{36|W;fHf_Oaw}8>ibTeCuKGRaioBWGRkHmZW2P5ml_S3%H<9o65%Rs0Y^^Vm@Y$}lKMsT87 ziLFF2usABMM^l9J)_rw+-Nef!gn$pDd*Yr890lv_@%TuDhkjRmY$Vpc5SpHx+>eYz zRitany#rQ_`2BA|^3vYKr!G5exeR?raOR~vOh@PM09+Q`qUdKmdiElq|HsvNfLT@) zZy!N23IYWZCF~-ZjkCK%m7E1!B;%c#J2Q9E9p+BhJ0uAr8AL=71VIszAW8;NlpvWu zIjZCwM9D$E-`j7`IpfE(56|>_yZdxkS9e!eS5@aZs^DIjp7irvPyx!uL#DR`-4 zL)uH(Bs70>m2@}R36p}H_TTFI%jfBy@tQtq&|$J(BQ>WB}B`)%^Qqm zuO*iWG{7AILX8}6HWBJIPeHHHwzMk1BaeORhs>K*uG!e!B<0*H*bRXlco+WJ>j0QS zy)~JD{Z}auT8-Da;&Y`a46)!1AZBJi8A3Z^xYXZf7cSwJ>)F4nc;AMg9b0B5)=fiS9p>HqD6ehP~6J z0$ejvK2hWFNV&Nv!!-$SyDIHg!c%f(<@4tWq74ih5}_v0LT5n8eKqzp#;Nb;33@EzH}N6{XOpY2RrR=oYYBCF8Rn38F{! z5v*G-xs8O0tT+WU@V`P_0{vhzoA~DK1tb-#7KuIo2IN+AY{kk$QdQhlRK{dc@SX1q zp)!O+-atn8#MASFAON=YLkXDgZh*Xw_G8n;W6f+>3H1k10_lk*mYIc=723`qCihbM z+wH_NW<&v`Ly*hp@>J@T?L4a2YPoqw$I9ZpT zzLy`wb~SWj8{JnFYbWe+h4*Fv^7pV8Rmax5A1F_L&WFg*o0kZY^>s)1st|uZy;c^x zJx~=p;>oo*<_Gx$r70d@IWsjUQpWd|U z93lwnReHD`IjyWoiV_U><3d4RNc1{Z+^BS3ll2Y*kELjHcto&i$^{_<7OOr~Agp|H z8oyEqpPD)w$Bz$`Fo7LD_fr94*eTOvgNKVBvFIe+(_8_xLff>Izr#6=94)yc9#)#M z;C_J(bC0lQ-og&EJ?W^kpV_ree7M{kEM|K}B%PF>l`mo58MeVFC2=;gm z2{I>fpHYvSl+GN$YR-c@0-$Z8R3+AViX>yIplxzhgL!%(nj|>9+9n6!I9e{5#Kp(WMT=Wg^BXGSZ_e$N)Q$=Mc#SRC@4%l8xA88yJMO8R?D+^XUx zJ|xHl?86^2J}a{*PKnGB8_$zPn>0r5ztrEfp9#Ww>j|Y?xV6&7dkad_TqfmkZ(j_& zNRkDcm5(=mYv~A-7VH!B1GV()(`YJgfC}usiSS%;n zhY>|zR3fi$(%ZMigRgj|dtbUoMC`mV6FVUN;Ta!+taFQ|ZjSM5xE2GoBaZ zu^DCiZ+VksUZ!w0<%@?1VVI}QnH5LtZwQr6vi+>?7YlF`p}jT}%vjzkF`|j48;Q4t zK?HVo*<8djZ#xY~^lNoYX|3_-)mAEh>*Wo@Sm9qq-nQfgue6(pdxBn#IZX)7@@*#O zF4qb$l@t|&DbLbTji3Gp*i_- z65uK^V@b-b8w8CrI8xe>;@8ah0MHW;qgS5p`LIZ1jM*9QEA5IqP>$52>mz{NRur+i z+uPw|AdAiIIyNzNSfs^!`|6j>J#Ac#LHVj6w?YpB4yA)g0?ibdQede}*$!nimC<|l za@vyPNYn(6XC+jOczHya>8#k6ImZ6a%AB-(BrQ_Qeo}lL^)yGoc35(nK+~Y7A6fi^ zPm4@CswJDuIN-m4?uy&~tYFf0RN{RBv<;VX#$O)(ow4eCyNy@QPQw+W z(u^V}C7sIF`U1(^bT(po2tlGns5?0#e~*5i*_ePe(A6LkS7R&Xg9oN>jOzW$vQUF- zGLE>talf$KYK+>n(Ofw{(D;01Lv6(4LbNQ;m=UKg;2PO_2jem&JXTto{5nn77XeKz z4EKiPG^L$s{57p=d}%>IddcG%l5fY9z&v?Fd@h-`GVK||tRCEN$OnIW1zoe?SVIN$Tg&EpG`$6ePcQe*L%AQL;C!EGhB z_adO~5CQjl;s~Wpd|7a$_g~%@<_}^C-p4$MMM1nXh~cVz3OYqYD#F>3lBI2&t&|(V zomMM*R-7xyGm4!Lf56MiJiY8w;)=z+wI)0b&N+?brU&_}?w@sQxwu&^K~~ z!=(bm0|y(;lU<5THJ%Y{e(0-77?g2Q0{vxP5Fz+_hMbQ%&%UM;c*^JIw074qY=1+w zxga;=aAjL8yJP`L6)HQojZ287H^n5|OOR_K{s=|}R|gq3nN9Q2uhYA?fmQA$e4i26xVKMHWQ+G2&Qv31NC- zDN76*CE%esX!CYlp1=J87SeRc>@6V_naRWHbwQMvHFH)R{#E`q=W3Udt}B*Wp{NCe zP^=HQ&{`3QiH4Lyn3mgCBIW5geuKu*hWB4r1~-#-5N=^y9hhK$gUiR{G!oaBSz+4- z@zNA^BpwuOVlu#4c^a__s3+toHd*mzna0|uCl*|l-{GD5yrj#Stz`Zl?c>0Q+Dw?~ zthadBj2Hb{Q7O8&j@V%}(xx3M_M|qc3UX`YFjQ+K``EKq2Nl7mOpYbi;BQyU(Lm9? zHWcFFgIKk`m|Ftv_^@UDQN)It> zeH|87tceXYZ4!kEha=p?f*psWSynbwna9F)mjHLPufqI=3pbur)ZfP$L=P0kkA%5v zIDklFdSQuRyeG5moz@}`gs<@A6^G76)+U)*7rzg#kA;ww{K6_3$3 zVg)5kA-4TilpNLAti)Kn4jPulT*5qtsBkgzlWH@8UI|e3Gjo#jOth;!ADh`SIePWC zX~DGWiFx_AX+#AY)8B+(8R?EMP2*1!ggcXDy0eKOccO`BDVl*D%S`OhDF{zpkekmM zZ^Oo4mDn(W@Cn0&`;suI+u)KS`JpX6y(lO{pmx~}P7>xmaxE>N6=;2gK#zK7|8RV) zq+7*eh&C@K%>Z%J(`!R8z?w6G+$t<2y$$au%-Q5Tr-Q(yf*4b7t?RLD3kg?@YLp1G zv6c{X1d5NgS`3tc0;SjEQUR`$MKm1`mYL-`LNnCHa42Pw$4ogi^e3Od0JHvp7eI(WMSq7oOswLY^0cASBcub z4b$Q2%IH>1n}Q3*+_Eywy`*UA5N4te^6FD_4oMG815J{n>ps7+^Kh-hop-#bq&R%W zoM>!PbO!%O^FAayFZ1wA`mN&1i%t+caUAdET&2rGElru^-O^X~f8DbwOFo4Ewb z8ah8VJ}A8E=3ul;hKdU7kmS`}1o9=l#fqN!5G6B@vi)%{GB1|e0xXlIQbi_?WrcX5 z*HJ^%lLkFjFLNeE;Al({B5_!Uy4XdjalRTBF`}!4bkQh5<;w9gdL23*P)dtQ7gs^m zZ;){FvLg*I_X*1zG{-Ww=n9>t1Pfammk=33M=8iCDw6^W@Es)Grs6m_2+|pU( zc}fLe}$TFFjSXf0zxOyEk-)4)348!%@n8(;IE<>V9Rd zrfeik@<&RH$%kHv-TE0}uDf57>R|j=h#ST#kI{(cP%OROgnXRX2x=+BePl}-QJBun zw+Et+=$+H?ZM7YWEIet;$#EdA6+|nMqS&C`SBe4+If+A*b#^4}DXG5Wj*;s5n_zQV zS93t#94roS+S@yUn_7%x@-)ZTPoUc+m1Co|8T0Q91oyPpurdfG@?iB#M>$IX%s<3s z2ZMjHlBO+(p}wSBSZxIWJpO5aJOB=Ul3xdaj3=|SbC)dx-C4C%6W~eu7uIp9wuZg@EuB-1O$;y zmJRC)$`vK=C#`BT1?FotRD0}7c?x|)Ke+KcFv}UZUe+krG_#Xn?GSog)L|pp4fEWgLI~I@al=AxBr_dDFDNVn4_k%~U_bEzu zVfNFD8uuwd9$MVJ)7}oVCDar`x=L@G#Y<&jW*H`K?XkqZ6gE+~3dRUlCIm|gbd8)e zw3=(~tygA+{S`|X5*#4V^aH^q1ib<(o8CA?KZtek9EqEhfGcM3SnbRD+YM8Q%rsp$ z*$>PgMARFwH*lOJH1m&wy2U91%p6>kqMwh`%Pa^2OwpSr0&Li^!2U%gh;~F0Us!;9 z%#o>Ip8D1JxXh&|zU%R;1Bzl8MR9byQ-FsJd9g=*Klwl)dNpm%?6^gL`)3(^5*Rsl zr~oml@E==xMJcZp)p=4Y;Q^`AGg(OSVzYC`%P{h?54pk5KAvpp>k} zmP)Zi;Cv!&>b^?454>K3qBzbJ;?}X&C%N^CGE-YU88f_4B6t+HGo-!k6f5XQBon@T zoG@3;ny zFb(SjFpl}UNzn2hR_dWRA*a~Puoyl`DQ{Y+iF7lQB)hn;7o63|R+DY9VBKl7&gmWM zgOv7k^1CRa*?9uZ>{2x9PE^mIJtWwRMcOUwM8K6*`eA%&=F?~vniD|@b#_CXkh8ss z2#`WYKB9U}LB51`r&5o{g~2o_7D`t0~4z&9VbPZY1twX#*#cN4zfBG=%w? zkFnT6ARdf%9w@5E?gHFg!bi)G60OY&gGF^*(<^2{l2vaL?CLnY4H3+3@u8wL(GWQ4 zN!WR81kmln(g~|do|+Tn_9Y_*E-5tco)GMhGvQ^?16REw&=f!bM5v?uj-z01HnRvS zCjiErLv*F%^N*uQ0!hw1Q-B+Q_8!(D|2!brWe9E2Q2Z;Wc>b43@atw#d4o&ufp|)Q zRv9kDmpYigVY;caXHJh!CHP^+?3UQz5dKc{T6@`UN_p<{7$WM_hGdSWPzcsqZ~Do` zSnf~|%|K2XO`a@{djv4HDRwokM@pI^a9i7Os~Vrc6&71shl%8oRR+>cKhX+Zdp6=7 zmG%Jfl4~8U>&HS&if&G`G6Pxaup$#{66->IO^CY1vpKW$w+mnmf(uQ1eBp2)nwTED zug7*uI){S65hgs^N=Q!qiX64__sQj*e~@K1&R5!{q=b~k#Os0}En#^&ZLM0-s_^Az(t zT|6wIoW{6}Ma+fi-Y!dXSu#`|K_02cia#j6oz{cFm@dR!?HNWXHoMG$p`Xkl4iGZl zIR+lVjL-sQwxK4w_fX6@hHMa;ah^1FPV_37g(r@9ai7xewPKVbwD0yq5Kjwi*I5Vl zEdd1~YmL7Ma2bRFj6ULT$AZ8y#1X~u>`A5kdxN(~v|I}j^W%y1mE-u^R1r-vW>Cns>lVkIf_}dl8nYt3Y z3-H%rW~q}9hRzj8i7rfVetv3Ef+>tZ@reM@hZiP(^dtUu7l~}jnnt9(5|Wp`RFzMi zMlSOv3I;CI($>1bk3qeGh>UV}Kwv=H{9^e7tFgjZWka1M^aEoB+jVo&WP4s(h+9uU zLez>n9xp2}ub*6HB$gi###F<2gjdH-C*f9iqKuR~`uaa9Al7=BSjSXBZnc!D_@8U^ z_ZKSe!91fYPCSE@Kh<6*c8%g*taK)rIf{E@&GQq_axMy`o|m za)-z{T43I4rFmgIbr?#p+pAMWFGge&a$;1WCw`Mo=BK$ym-zAb!W1xZzGmD2h0SvJ+t&9lLcn_!Nc-sTkz*V;;|_+V~+mzz{*oI zS)8TT{Y*db0W>EK`j(6Wo)P9L&wI`BoRV%C=d!fx!Vx3qfkJd5Y{5<+|H9SKNzA$k z`~f4Ze?HhaiQ1BH{)U3w^rYlZcXv|-X5PTU3KtIl&8CU(5zrAQ9 z6Zga;N;`9d5M$N&)g=IQT(2&1Ak}vEn@bJRQzi(j#!&2W8IZfdvc?H59YkLc>dLW+ z!vUd8^O{;SCSA_&X-6eg!s5Rxs%S;arN=_y@;0HFEtqueUAzNnaD3vSo?^WwcQlnZ zbIWk2ue#1JYwbgc$*Lgv>@Ot>Q*{#FMWLQ%T|s$sWzu@_`P&4!N)+ZO#}Gf}{Jy}X zb7J@Cj5~#8vT3D`rv-Rc@fZU~9yCTtKlp!t;0IHRdp(++w0YfEf|^o1^V=B>rKwM@ zAqw#w^IrwPm`!h)8VBod+R4HymuG$dRzaSZqiv%+VnGtZ67OR$u2I5$K>dcT4U)!@ z*MPYdXjHJqO-uO%w1MFs+$HHVq2@cjXmbe>@if(8X71+hie>I%A^wt1aBwL>4TPC8 zU7Q9R@nTs_rvdKJJ{DrSH;CV>{;Yc~kcVB5^YW4};0%EfiBq`D1Y*~mh7aYC7pExg z&kt5x5iuFSk?YF3YxB~PWryp5Amyy7ljCRl+eMmOw{v^Z6MNnOmbq4Uv&Sn<{+ZLK z#~u3H7h~gcTMke8Z~a0)u>)%&-1fzpHv*VE?KqMmFP-qm0)fA|GKxiRDv+dz=IT(0 zS(_DtNZ@c22hIxxn?%G^L9}TYGKu6Ctana1;+Dme*Nsj>C>s;(w%{EsH^RRuqdSS1 z#mVSTq_rks)U%WRc5bc*xFCqn^RLs1-M8AGi>6@Vr}&96D9BB*x3$1W1)3$%jmEBj zB`F4;Ju9xz-+6Q^1Z~5bqY|$R%(G7c#?k*Db&IP&@EXFx{um{-nIjpt#idGR#^MgG zJ)X+|$_&B|!XNwI>Kw>|ob}-Ps{~~Jl*e!lGTtpLFTgU4^fgC9ol~K<5zGj1s+^hN z7_Z<(j6gO4Oi_JU{n`z>G@7|ul6_uKX?IUCQrK2v?rlYd?F49Y?CP@xn)s4OVBtRr zaYqq&*=ZKNy#RE@I~1!4$aQPha-Vp~-+*!lxU=Wh5kF>Ot{RQG^vY-4St6WIM_%Cq>p>=^iq=amY-1)ZjQF&V*hw&F1q2xmVKpC#T87?gfJ@su<4?#uE3DqD8Yf z4pj6vgp=nUHu3~xz8|0%YdfCuSf)@aX!OZR?X(9-xgFf6j#27M4;7_Qoe#5#u!zQL zLfsUcWO^|2s!YVNzsFXBvsB9GUT~&B@7^A(t0=28A-aZR-LiHz&LN3Ch)~}aYF^VD z8O&n%e=mWqO0gjrR70L*{X9WE<6&~R@i=MBW8mfqaWj!D`t`s|9A4(a{#YmW<4Qzk z^pv}yMB0b-Xj?oWMARfSSc>}jv;dl;!NP1$`nfG00fPEnweHwae`kK+D~S3ELf$LX z{DrZToucUPpkVVj7f=lThL0Bc6tYU2uLRe71m}FH7JA~=#{e>u@K~i5o;gndnF#6H zpE&G7Oihe>hKTE+*&xu{8i5rCiF)CX9p8Qu+!TXvhy||U5C)4(Tc1b9V=txrfkt{y zGJYb&t-#$iMND4mDWGCHU}u`F-PRZCmf&>C{-_53P@p?Cb;`zk;35H;&AF>dBFo(q zFdm&vw_PheU3_EQgMgvitVGj;m%c9Q6v~3HfX?EupXW67C(EvnmG(EWsZNqf%=o*@ z$dz7)#d%6=su7B`8HYYg$~{CZL|#a)c*hI%0JAm`IY0&vjW|iLX;zn|4N)wFn01Gk zPk&X?l##`xEfgQld0I{8{huR`$;{PL3YK+>AZp<;Bi8ts`#i}!G@~`e;6J)V*dz$w z5w{3&EwZQ968oBc>z;m#E4^F0Wg)(HXx9p;;+Q@ zLR}Ikm1IqvAQPT#xx!mWxDH~*r<}QKIvg~((YRTdv!dRWPAP??dQ`B#jrBP<`LC8= zX=O{6vb6DP@nO~_f>T%G2PMS4ZDOv2Un@Q;3lt2k+T&zlta@lBr^V??m~u=35t!BE zJb_Sd#`LLix&DS!SdntSRn6YN)|;+i zl$|zn*YSc(niT$4j+HkG)9AGjljyDDclcyTYfvm%Vttt3q}!%--gedLJO^X6Z(*C5*!Mf?V_TNPr zJ^_7?{||tt?Ese}FDWT{O`8)7zsuj5y=8r|xY9JwPdL)g+3x|E;M_C9jC8ZD#BfsQ zdLO}Uz4u*Ef<0DSgE2*b3DgPIkv%v%KQ7Q*%sSWB(+U?c$u{}`I4`^$L#2@7u#!-B ztphtgtqch=rwsEx9DG8wQTS>CZWS^T4?=xxqKe%J6fU~j=Pdj~fvz{5X7Das>BF)C zOgFt8jtIW0P!A1Z>SQCig%H>0N0M~&a=k#)kkj^9eDo1%$X#b5Fk;JpLedn}Z7mk{ z1fLKFnOSZ*!f=lLPylsENYGvRQlF95yqYpQuF~Ibude(=ngjn^K&^Ex5pB^W$TTFx zA^w|5EnF9Bg6km2y1~q9#vx_-UInpaP>&VlUMNak{6uMYeyD;yM=gF|A{AWD(7>G&<(neY|2BAVsgzo>J-EpdmeKiIOfJ(hn}pj>pMcC)tiK0J`%- z^>)^f!8l%!_;=RqSZRL$J%=F3=jd-!Kv%;Ox9HcT1&Rt}GKRDkI|=f#))IT?1P7LS z?D0i@F}V<;`G+=@;TjNZ+Vo4MeX7#q$7^IznapT4{KJCa?g5%enM`pay0QdeE0hW- zJ`B2BY*bBM8egq^uGGP&kryPjeq|wW^A8gZ#}KqK1H#zVd*tRHH!Msl@1S-kd{qcN z93uimEU_pF=rwcJAJA~vO4Py_d3)lFt z1QA)=txow!AS}&cn!^k_;AOvLKuc_j%(9UH*NzH`H^pKzA>$*43}y-9G@-e}(g=vV zFBC>QrJZeyyObKA5h&H5$aY@_qLZ_DGVU$?O;Z{eyC-v=U6urJFXU&RUeSnq1(J*Q zA$Mcfbzhka?`dKHXr?|Z(7ZA=-YbjfJxweWm-4Tbc5D`QR3i5emqYH}*ZK;`IP<2E z4$&ou#Q_rOWtnG`q+cALSg225nw04QKgnrS-1iHiG!(1fT6(yxdl`UoQtL@bEehXS zwkXYMPLiBl94-{f7qoTN;(4VqtMEJ{*ZQ9eF>Q3aX+e9T0Q3;x7Y`S*Y<{=I(iNeU z6e}%9zB~|3MOEvK)xHX(*@ph(0 zpke~ubzN_zHRNs;7Zc1v)pCy#nfhFAvVoys<5^n4?jO<06(?ZY${?O=c!i>HRT|37 zKqK>W{q2@Z>!mPb7p&s!=`jqpfIlkj3cIV=wPRDYDu~<5C39Bf+#t}|rp`zK%1-|p z2rM?nd(OB@wyZ|d9pW}^v|ZsN#)Q$776jg(=f7vSOo;{705`YKio=xf=aShl7M7iH zra(6wT?c2;dORY?AK>a^4AoF0o+-;}AxB?;jPRB~Hxx<)=8~JwcxIupn*PXfl91 zi8m6)5IPgj5Xi_i>X`Sk&-Jb4G)qOJg4*g7rQHz;&&XVl3Gt_pU%4i$#4cY4GAGJ+ zko`fks3|n_9P?A!ZP;F*<_wWE@6g|TTOx?m1<$QbnqErqoA<0)(Kn$rQGcU3<2n9$B3>E2;K&E2%pn#4RLefUiN_E=!$n_T~17vv8}WQ2y> zwSZ)zR@zzBP8UFDQW(mbeC?4Tx1I&aS33!-%_91gk|qPjV7+?KlGeXC8`NWlb&GxS z6K1J_>m_2>5!~$m%iG6&H`o8zq^rC&;hdmp9ihhwBEh0Jp=LhUgU@=-?KZH(YnfBM zFR6|^rv?UaHmbxcf;>7}rX0F0$BBUM!OR8e}e=AJcszeU)SI6EDyjkhnulu3CXHg9v26fP>mVZtzVIcUT$1k$-#EtBIO{mpb~)Q1Vo`ERA%;^eWf zUGu-+Dypauw1UNvL-{#dx{B$urpJq0@wb^znqnSm@eSTTw=OEn7sJiCa9c3CyK*TB!DrxCvsXfj}z$5vta=8uBSApL{5MBG}IZ4-+nWye1!fW`j3-gcDC z6IEWFtot_#a&x4|(EI23+IfDcGHqy=1{ZE4Y6`N70W=ktcH zjVFMLQGi)F^LcUQ^he;WdM(cSHUPBacARamFYZ@bTS3e8XxWXwX)*7S^u>@8<~0S8 zVk&Uo!?@roX#LzN{O9y|)S!C(Q${uehZCbPHAQ~mL%rbL9 z%{&Cp*8PE3uE9OQ-Ai=M$&#f#5@`0stE;znfWX?r%9yLwV<~7YIoh5jG`EL4Jr?o! ziy%{@6P58$+$F%Dmi|KXCB-rP>UWE>G82+Z?qorx#|RI?ur6LH#7%*1;3f9@7lpY( z0z{|XCRvevYcFtrnZ1x(l$wA{Ue-#gyM1pEMxuqi{}cVqNbsi5%(#Ca63|_j^$h7k z`xYhJDoyUJVgn&$fTQ)i)x!Ic%uJ3EsUkR?DacKQyl7fjI<6Du$(5op)wEu)#_V5| zM#4&x{Feloz6vy)W#PY1K(S`fRS>l%)E{rf=u^RfmN<~&bY^G_mF8MX_;=o!kkx5C zCdk#xZZ16&)D??-ugHjhR? zm{l~!D6lZ<;oC;3b@Hb#s&t`T6W(1PTiGt54+Y*&$w z$VAMh6$L;h2X;?;RMOPEESw?MBETt|kHq8}5N%0sy9{vw8T$z_A2iU-RtO6Ba5s>p z5-Fassq58U0@PfjhhI!5X=K^piE`Hq=2prYn6oK&Aq~SGqps>$t2Ck5F_d|ksD-eT z;Jm=ps+}^MZI4@&cYLbw*fS=YF{cmAq-@C2u0OsZ#GJ*ZJ{(&s%?ROmjtgIzozmQY z6Rp3fl(Eo0h6?a;!P%O1oO|lr<1HVbBG_zxj67pBini<66!iwAvsUUWg~%}5i$}$ z|`JzO^Zae8N3Ac>dzU-Bu;5-thKd*JM#~@iXfjnN_mEr!lPHC;+2vx0P{(JhyQ)A78 z`P(d&EYed%0Aa39FPPLhQCu!)oE^~^%9qw4!Wqs~+-qH-EN(bUXBtR4c(qXXTldz3 ztZm}tR|Jo<2Mvyu4gqjGTJ;#7EN6!f1@lL7@6+l*$^OHOk_4qf&vvH(m!zqFl6uM! zU{Gg(_hDk%ktE!mR%WRDcy<)zw!(KDpE-_HW4)t5Ez_j9n~rzO@-j<5xFN%MGI_;K zpufvq7t@?zFIy_3yVX@`w)e-j0-OnBcbjS3V{8JEB?0xxG#FlLu=oP#{eT!h;};8aF-4V=Y^_g~qPCfpS6Ym+Oc9-m5JZVXSfb;MOF?QRVBSG8goazWj*=s_~1 z)$+^CDvr1f4uEA@>*;Xt7sTY&nM`8-2_#(=3slEYLrUybP6RZWnDPS(0}N+Q6Y7RE zFvypM`Rjt*TjJ&tSde9O{*%EVRx5HzC6-Z&&oPpSIWg-LlKxz-1MYrXU}9V}hsT^s z0@|lL0>1h?rKtdWZmFBU_#=|0WkrU=$l*yOnl9WeX;siy@!XWt3QT8eS=ktK*U~vV zy@FxX&BA$=lZ~YPRNP<^eQJ^i~D9xpB1kw0dZ{V5+5k(&-7HA zc&An43#S8#brtO{HdV@u-BTN_#c!3SGQGnRS1XZQ)P>=N-E{sb)SYW*4x!Hb8P~TnPC13cH{)$O$I>Mo&{=JEA*|; z8xz4MHs~x4jYy!^E2vBS{{ejKB+{fG#^5*Qi46)2_g8%_w~K`h_o&g> z@MmD2cQ~kEbUIf+9#y?9(;gQIfm*$?AG+*p=VyQQW%d&SEG?z4>*V5nz5N`1VU!3- zpvPdkb8;X(02Dv@IluZlQ|HWSA#UH4b6qv(I+zywvqJ(Vn)bygqptSzfgP(b`V7@I+Qv0o$pdD1Z`Uyf6tdcjKcU z33rvu3C>(>=^vL^AJ*;=95=q}it;Q8Y@ChiJV_FZ3U{NY#STii(VDzmo^jN>M4&s$ zGafqS%y&b9bdwKF$(MwX3&09fDF^<&)(8abd?hxotOr4%+_J)wmF+6HR z$W2ThZYzprQwHYwl)n`qx1|`9X|+k9OS68UhT|K1DkH5UB(Su+9c4N7I@$Th(E?3* zu_QAf+U@`{t$J$BdR(it$^621!ur(R)f)r)+Z}I*-DJcP zpOzTPM{H#1{iF9470cR36@Pv|5cFvbX=RAn|L0`Vq4&qE2b^S2T65}@*g{Dyac%4= z@ja#TvH^pk3yMnwnM^FCcoe=Mz+IDTLp<~#X+EcE!_<0LN#`Fz^EwojhX7nBl0cfu zH9>hO>fJnr`*2Z)V0_4D-xL6eM=Dsg9ifB?g&nGVJS&fYjIXkK9x9p5b_;brofJ5= ztp6y8Kb^EbtTs~xnQK{Jb#upIcc=uBsBz;YNLvgG^hfm64}nGHotRL%mz=zGg*sAx zBk9}t#t#M2dPI!jI7x}j!6{fc8Gch@dizjDIuo%5f#y-sRpZGrA1YRzRIBltFf$y6 zC2*)V4j$8f3^;Rhy%(>;0ReeILy_v>tuH-MGzO;$DMqVt%#&b@A2h=%KK}9)Nf@F{ zy@`jEG6j(fG$CTn--|j>{zy~O9xn^?*X6I?6K@Id*KP17#$5RyfF7VeY>GK|uT>(f z3({C`BE+mw)wSZ>GRLTcM;s&s>LM=;#}AayT7XWPHRSWMbdStpSNJDCd(VM*rJ8Cx z;cm$Q+9=k_n@fb-CaI8?e}+6BI3f#}#ddtIz}SpAUXZUMEFb5336RZO1?f1~Ej(?= z)7g-lO66I=h;?#T`x8NKgyN-eEMN0^5U(sPv8xg;p;s7nFnLyx4-US|9T(3((+_-0 z_jUC+<^@s^g(==cSo7syB$=0Q?hl}gM5H@}<`F_KR9~XG>LBiZ-#WiAak301pvj%Ex){Dbm0&ur6mtu{HH-+RZ+`ywx5#XwI z;2~lnj(;x~nWTDd#%U!$m<9xlpOyfj3At`s;B|^*nqwfP2Y!w#p%hDoihnG~ALHM3HJFB`_Wi5CYy`~-;6h>JlK`2pHSQ5YSV!&S`2#(;X!{K;b2~V)jN_V`Xq(8z=ftj1c)wSOT%>2-k{eN49xn3l0xy8nA z9{~B&Qbot5N>gtuYi~6S3WHYwQ(!P>qgHAF)a~2h;NZt<5{6@y)>fC?|4p-JK37# z3r7Cc2^gwPV$XNQJHoWE%_0cmil32i1K77&wdV>j{UpK3;I$Eb0_iz-uS8rt_`gM# z6eV0%UvZs4Xl1p0{8>rY+mNqE+&uptsR}}IA=~^zDcXR}6w~(wNSN*LzLN#X)k65* z2tkyparzfY<=LhKBqylEWoHY|T!vdx-LHwBBh-{pR8sQCxq?iY6v0P(^!dWPx~7X< z$STggu!K9S*zhuPx&-{4;5@BhXAJ&f-QNrJ_s6Z{5duG75EQ|22KK4l3-LEZfzvhT z9#GP?B%g+gR>9td$MesKm-M%LDyI)rY`(63UAQU4vN(&lT5l`@Ld~d_WMTQXlI|h5 zT*yWoQTi7JgT`3<6Zc@Tq6Xd@hDl>LAuhrLOcr`?7sdo&l~8Q7=v>IS!N7lGtbm=q3qyf-eu&BbS+);}-y72=QK(agmk4LC_yW?_ZqLH2vIEXbjQ z*Z<-(A>L(KQ5AkEdpBDJtThig?Ga)byFE>+guO&(Oi(K)^O1>E@dB?hSk_p zs6WKUi0Lb;kmJhY@-T!`I8>b?(2YdR!zsEqmYNI%rDhX(YgPU2pRsvPO1-uc5_fU0 z9zPNUsavZkUE1RbCH?Vk-i1g3c{g7RjCD_OCZk7*Ohw&vvZ@m9yTDLlXfh7QaqEJ) zZ&)36N@)vSS6L6#qXh-(DBG(-a&LK*oaL?=^MtwO$;b#%Cqd4J8XF}I<_`%o8)0w6 zN&CDFC}*Hw#u~BDh5#_8A{I&>IYVh?C{j1u;6wRWy)mZ7J*u>e*Rg3Ij18v%xE&aq zCR(gJ6-ax13(u#$tAq=*bxG-X*0cg3>{P1YI?e`qxiRnnj4@=Zp+6`Cl1JIo$Q;nk@z^LlcyBKTX-w-XR5Q@VU;3) z>eBfY3G1mO;wgaOTEJ4lmV#oFZOvV%s#Pi@Wp{^StNGg|SOJqo52q8PRCy&U% zF`MV|b|5vr^|K%mCeISgL;9guF)%FExh8>bTD4!NAqez{GP)(5b+oz63cm7aSzMlZ z$rU%A5NP6X{!Vf0*WIxw&3q;tOHHvHTZCqIV9?kgB_74$g6SRsabWW+zLVyVzDzCp9wPE(uFe5RL>B# z3NuMK@vwI2g83|gE{eKMK4SmM2-K0wD=c%tw~NY{ja692l$(!&`TK3VfSAdo>KSsbl|8;!1cfb%@JJ9EK2T0E1^IzALv3i5|wBQCCD z>YgC}5c>u)>U04v#e&{cl%!?%D)Mxun8m*qKs9pcMHS3J>G_PHGU8H8b_p-n+~1?5v0noZ$Wh8RCGly2&)q67oyVpS8}d_df^SR zud62)*Kr&i5s!8d=0RqiXCg?hk)y#s^@FD#`gI)@H03wiuRys~PbSq2)(?bwq3Wtv z=0)>B(rz=7rHqZ^3*RdsnfzB#M14(=nL$yh(8Kb9H7bCnHg0WPVg8?h%rZkwoc8qw z-S=AoJ%=F*CaV_P6YLF-!+yHh?vV4>5z6jC9n<@tZE=(kbHltEk`^LfoG#39Pt$=U zxx>JsaQtR%{Nz><%ZkS8fGm zqB&q8;5>ouGwLNmLvlazVTo}D9!&?LxjjW;MqU?}nq2~EOA0QMPU0W+f-plxX_~fp zKS_E!gLC7>`rBXPgvuoNS>c+40Ngvgtq8gm`wxP+)huP)&&gaMfq75Y z&Wt*nm9e6%?sJ7_m-QT9yP)yzDIzOfKDo@+&(o8N&G!dEc}uD4eN>sj#p^=N!Ca(Z zan~s7MelA8jmXfxk!4A{Nqq=FUi|3F>!_D&uBQ!e*V z+rco(zyC!!sBjER8|?0?(Rt+%iV?6*;d3W^XAr+phWE%4V*avU_r^8{7p-B9VQcP> zy9JpHY)^^-_J>12@~B{y!kX3=R}1qvBb#&3kgzcIP(WHTaq2bW)WblbqGBN+i^Tmo z%@Wq5P!mfX&hI9aw5-x2%j&XU@Hj8jdy-4wCBi&@C^{H{+rAIL_COSN?pco@;mv-m z*%`+u>17ccH0?q8G_d`V{NN8VC{=CV5IGX;k?CSFS16JZ<#ZclO2FMue;g^y*_c*I zU_POQxWh?yNT-w#(}xADH!c)p!Zx^9gSGD!V*cu?jNxH?R8ciW`lSolLJ)nbV13O2 zY&RuM_Q84+L@V1q0>^a=@LHR8d8$7{ILtkJ>g0%{`P(0_H+YJUaTemfb)=&bB~JSx z7+>Rw7+w0Fr=-cPAkFQu!Lb048>RB>Sm-zsCWO{QXc0|8>G$a~;t?fWdQ3Ek_XRK@ zl5x;yNsWTg!kpOZ_@btPD%*a2D7F@4rq)_JIHWx-wh`(o`*?L38#e)TB6UDg5F6=N zPA~?4rHg7x#<9Z2d!l>llLeR_xQ|yD*B_q<;_f8N5k7Z%33XsCq)W^51kuB`v1U)~ zdQwqqD|4Dt7wIaTA`oo%4kTWH-(Q3Re1U^7mN!@(<)FKC4F|kr( z&S@YfM~#gj7 zXKrx2;v+$DUoS@|y0Yu(MJ0r>RaoGko>3s^R+t~L*iS(c!%dkHt0-Y2sriH4>7mcM z<;()L`yI>kKwNiLfif;oJ?Z+l{~4%ztJ@pvxf5t&Rp%krjVJ{CP$ZZ^L-b?GLhw^r zXTyC0`QJ1>K(I;CO2kQAKx#s~F3J9zt!p%e>5pOk;Ne6lMF45Um>@TZ_gs1NS)0RE zxk?!WvaWHd|GE(Od<-kp{I=7gXBUOJi0GHWVtWh;G&`_urSL~_Uy12rCq-s><6QE& z-AHH?6K(d&L;Stk;HBvF_H~a-ZFm9tLfNArQ=a(h91z@sWA%ww49bDll>uM}_ z0f4)uU}&b)h63F(-7%(TD|Qp+6>@a6R*ONU%|AFfwH{mim=N$1Y1e{;Lr5o2+d@v4yQM?<+VvDc{y2Oh)6Fx=ObWD(A~LQL1mEEK zuH(hy6Uy-J3YwX?%pXz=J-JD@Tu$2k?_@GmbOPI7Snej3qLyCm_aNpG4##MuzVQcu zye^<%=3O%hQ+-0m3CH&lrnMdQ`iD$-!ig)EPs-~L27h@Q3%LPk!@N@iMM z{3sb^hwFj7YVwe`4LcZvc{c!?@36%qmaI1RBLq4xW~&Y_g82MDu+&hn20L|vT?L1R z^vq<=>lUC3mFWkug{jqK>%d4zG9uqPscZlWndBAFh_lPAjUF|sO;(wYmRXsTxW9=l zSGlq105KdI1WbY#P7ugg@cJR4N#TSOZvyZ*>q-sEUr-_)+#q6Ov*4eKJlqP;oYNA= z3o(Ne?F$C~_MZXVRZRJ0BgOi)r$Bd=NI_hu6Ndg~Ack>(S7T%7FU9Y~H-V}9H2!4$Rb4c&je+D$80Gn6wfZRDLfc8$FEC>!IZ~STr9{f zW<|~qo2+pQp!!x*XiQux&Y8fvG@t38QtH;8(6@>;W^F*6Nh45t}XxHP%xBF*=1P%a_OwScU4bqFi zKWsXGbCz}EeDNUN)18t0Gfzod_4Xn;-n~N`s499%+chgn4dN+Pf8EQl{X4x(8HyDTnTd^*1a> zxcRm*OmIp0bFGL*uDJRrK zb~~)hahL`;o>%(E#E3Jte2P< zgn;^3n8{RSVS(?z{1Av)hpmVx`ML;h73eQe8^q$SQIJbwDvXvRt`cId?BrgvD^`CP z0Q#U)>5VOwaEnG2fNHKy5^mzo5R6JK7q^lGNEnmkfhNieU(mI7TX_dP?|;(BYL+O|;| z!vs&iKvI(gp^z)UxcEiVCL=LWhmcTW@=HZMbA3i>w|6`mp9}6Dq{oq!N1w$sItp(-VSd4|aRKK%tWySL8`E0cDx4 zp@Wc!`#&f_Oj#b`=!A~Xok1)cdH6?_x;L`ELLIV~5cfv*PTZGNV_#t=Gg49AdW!=y zD02>!U>>_G%KB1x4<6}$gR)+!M(f?NypsMP-c%fv;+sO;D&2re`q-+32Dp`wLBYyz z7P$!ANYmS5?-C;FE30Q`{Prz!u?mmyrH}M?<`8PHRIb#d0Rp2i&`6T zDBX@8`JbXb83Ckw5H&yHCIyeoHWd2s7-cmhbr9DwF#`hB34-&wfKDU1Z^y|3={;79 zaj*U!=ONDR*weC^{-02Hmo<&+UaYU)73Pm}^5M>*KjwQENZUyZ5Bb!UfE+0Gb^pg$ z94f>maBxd+4lMLu39*eaD0IcV<;Uch)tf4f7Bx44ex zF8VQ`+A)h9mngxE)l-LEu`(K*rJ|#AK~5+ zJeC&3@1wQup4eUqx}%k~+Fgl#l!h1s0|RlP{>~1xHX9wpQC_zMi+si}=1p!A_1f;o zLQG~%XYdpm_`-L_JCKYukR4tsA(3eimxlWmAeZ?=@)uG5bi4VEP@eG&QAyN!s9rzF1(oU0~DejIRsxr}2lxP9(+6-%;o|MbT8t zqb2@Yf^uHq6vPqxiiKQ0DZuHEWxu|KL20Em(|t zQI~v;{R&8jXM+8b(o*d~-;<1>b zY`GmgNN7*w1Ues1Z;8;GDxXM6AwU_R9T1yMVXkKg!4(dj2ua>WIE_Y`%f4$}@hd@QIb19oNRzsU|4gX6!!lg2 z4v#Ry-UvVs0zqp>V%6nHGih79Q9P`*d{GY~&A5&8ZbE%Q%aIM+3I!at*8BXg@{?yL zfm#x6>`P`W>4w9XRbLd-0! z4;Kru#8Abo?VPWXgj0H|IDo`^N|~S|sHYln+iC!=vadoYDQJAf>OhQZI`C-a)E01q zKo>*@j*G<20&)KX z`s7>$p)Ot@)C1M2Ye#Hw(*4RM!p%bUF^N-m2ypT8_#V#-bwhf(nkXmY+Z%wm?Rr~# zNO81x6Xvpg$cBi4a>Lp~Xy#@-b7iHxpCEr#@fx90nTK-)lOL0g3@PX+i?Z0_PC;&k zR8cq_KUkJ)t#M4oV&|PQ6UPQ!bj;q6Vs5IQM&%q%&~ag|3Q11k+8-*B6- z!;|I>&1kP6CN76C6hSAJ8758hJ>s!bif=)Cy=>GLhff7#%n1C=!qtekm4+q8^eWc6 z)BGbF3wq{xF>N~OJo`|>rKiSY#TkI^0hu5O+Od^{nTD9^L{80%i>4o)&g_&%+fnxD`RtU`)K;e;= z-c?%wEa>STPsehd)mRiZnZ++|40rEXfTb$87YcRQt`LzcaqMi;d1Md?Fc8^i1IHtosN^0WWSU^FGWmI4n*v`@M?nn861{7I(TAn^P<9Z zc?j!nD99Be5pr3m;rm7jrVEnfO?O*aShj8QvHXW1(+?R@RsyKWgwa{7t@YeK~RO2aJ};$kcD4H9`wuu0`po$M_rHyrIVeLqsvJ`ykd4X2jhZm ziws07l9m4a+Z70g!GtD=C4^|jojrX{RF&{7K@i6D_-O&mU{pp37;h+L!eB*?C1qmN zIXi&5W!>_2$ux?M^Tzc{uyzMH_z z8!U5RfC8ed*ooBc6Y0E9zL#bsEJQmzO$M3il*aWN{` z{o-m|uJ!deQ<$5<70aj=jcA7l48Fa9YUOGpaL^+COSQ5g0GR_Ww!jL5HrIh z9?{usch{mfeRYfao|rSp;?Ef|`&W~}mEK(kSLTuW~WVErtVx@YAWDVr#r34xg` zn*Z2G5be`ripUvFC0#jtR}GhW7P@Q8VjSdJWtwtpE~XbW$?>d`rh2+?lN#}ay}%eQ z>56OdoKo&4?uCSv*t;l!;h)yTfw)ANJ3-`XRNrYXY_Jcg`GD{o%;lDS0T?@U#VL+H zF92jjRF#bx4^);2Rdf7_rT23wsWfdv*pd-S$-#(q{stDv1VY{`B^X^6vi|O$f8e8%q@^|(!a#{o5BkCUED~~t zNY{%AHf0sGQbEPzL}j%orI$!$yMW5H9_I>oUUdDa(+ER*k+96xiao)Qzga@OY^BX2 zJ}FBof=ue<*S<$duVS;~7fN^#*!ytjS*i_y(M9xvxEr)PFTr@G#NJ9|F4G+bLjSoP z&Q0Kb8gOe)wc}dWjD0%!%@pP`Nq6qts{Tiy>%j6)i(RalAGx$jwG> zKwG0r*+&GL%iAk>&_9;bNY;r?|54fvL$!c&>BT(-(9IUzjQRTjXlZGn5pN5E-?Ivx zCE=ca5ZcbAN;6(o!rV{@CtA&`1K@5TD@0E>ZRjy*LbTx}sSUTI%%_tg7O@P<1P~d3 zg~yfCq_OU9xDhYNMvuz_Knf$8G+ zWmfJSdfQvmkl%khBE!>3DYiN3XG(l!QmQI!#IYMOEIf+21 z=)3WE6?RTN)|ag4VDeYXIy-CjI$F9(PM;GitsIfmn`FAVx_%g62T)#Ug!dL^K^I%U zsysflxpUy?)GzR%m{pNmRv05~dN+m%yvOa;VEj#>hm%EQgeMLTE&yDEdOPB~0#Ki> zzWVC%>qGe4tj+Wrju(_R$ysydoO0Kp&OtB@Ss_K+VWhPIu;kX__3x9)WI;D6tI=;B z0Yn2DO#%x@vG!d7Ccegp(e<$_;r@2B4|=x#Z@x5+`r={2O(a=1;HWbkX9#mU8%=J4 zaC6KEM`lh{x~e|qNb;CGJnz&?XiQpnrodboO-T~c@kbR{a*)AxFjhGlj9ze4M#Nf{ z=Pfhv96r7nu`@#jjFW0a4Sh|BJA}RWKy3YkB2BEYTHL2Jq~jV5L2u5nq|CO+K5h6( zN|N72s2k1o+YlBi$m)+CS7b$X0Q05DcN0)fDcUhOM>wx5NG3qZnEDA-;9l!yg4l9x z@nng$VZWV(>}P~|I63K|;cUj+C4>=8IxjDCe346r7jo6SQji9nz?KN2T`_h7Kpr5h zxrirurVzGQH02#gzSR?50(uT$^E(uW3vdZ^lX9=9o>XM%#@e|n?mW53(owBSyT08i zMHWo2Ik_KmY60+?8kgmd{-`L^nv7&)$$AIL_>&AKZIbMHUcIFCmVc zm>l+C@9JEb^XU+(yZo6y2J)P19qvI{bgcjng6{kXC(m|uhd>X4?B6ha>4`VXj5;fk z5k5NhbTUE-g%Os`d}rJt#P#VeL)JrteHEBb#JG$kxr+4=ChGhNzgRpS8*Yu`lytuk ziFkyGhsp$*`BGSKA5a!O3)G*Xb-E*lD$oC{C=K(q*RUy0kXJsr0XG!T2|a9rU7UD4 zRH-A$3o7njbN`|1?5Z(qfr&SZlTeo=qhv~uvNK0G3W_}Cx2s%T$%twKG+0AF!S&X^ zl`GFgtio}1S@LWU&&1BkXiwa)v|HAP*;hI>$MXU`Y8Y~HW53}!0H%{{=yl7pry#c; zyL825ti+n<7L`#IGeR*D*ZW10vB5ftYN-+RUltf9Rhnc3;uB`VQy4*0R)pP%8s%_LAE8^H{uJI z7C5gu6E;3wBUX)@Z_qxAm`NI*U)c?Fe3pZBT5%KXHo&zg&!qiWPV8@7%JO{mGA|8CXUAmOx~Fh<%GS}5*h_!=6X}`8 zj!JZu5KaX6iZv$P2l#uj{dcbI(%P7M1MQM*s zU0kB1sjPr9LtQAiv?yFF+$~~^kocQ%T?ujL!@UB29+wxnn7*wlxY6%`+(?YC(PqWl zLg?sB;@n?!Me*xE+S238zXvhR$t>NA2)VmZ6LQkbd=&k;FfTll2q^N7l9}~%bRCMg zvZ#uKJPRYfDD_xdsHbap9pSg*D$?#*b&RVY)Tn0)avOL$Q2I_CC=%*6pn#W&Q^S}G zb8$j^foA-ytgi!KEX}xc1-WTFFGq~_IA6#(yFtC=HF#lJl7n@MAN6ZN=2qVqh`XC{ zS(%ejVspUuuB?&A0eAzILw0;x=0ow0y)=e@*A#8(uGHj{-Xp|I6LyRYY*T z2>aq5L0Q_UR9a*0Ye};$B;h}J8*t`9OfdaK*FysH^2mCH z(*{;)yssGdRrkJZp_n%Sd#1f7w0QM>-0%U|ELx22hq@RL}Y9*J9Q4WJ&bA*i{pV3L3WXY z)_Wz=ULzn!HP(25Qh7c!kj@dE+hVf7+?Xz|ve-jsmynE75C>wl2gxN~LEx;8#yd*7 z{~TM9@wr)nTwi+(czhaiOl5`Pm@*NSF3^)Rp&kPK-*MY_K9j1O=m~h7G$&$xZRz5% ze4|Aq=<}Q{sI=Wx~cv+5! z%L?LfmmqhGhvJ8-DQ;rScmgy}T12i^?Gi1*GGFl^EBu}ia}dH6Lur2?U$PSTQD9PiWv>}eS%yyF7h-9=DlB-TZ*ckb11l+q8};qiWwyd#q+{o2hRAi1l{Nvl5WRn zKaXW>qcp8V6s|Ype5L#`PFcJJK&2}_3uZ!bH&2YWc=S0Cw-Unu4jqJ?ejbRgRQRFNuBfNhj($N$`tQF1NXti)VOkvf7D+df z#eywo`+pUcXupj^mCn3G_>?3_w7gSbc$-R0hQEaQV|p9Kp|EF_pxg<#d0{jYmw9@TUR>pM&*Xi+~5LU2* z^rqB(|0V$gP%YG$?>$mb!>W=v@O_eQ2<9-%^H}Z!5NL||H!7ZHtfMqJ}rO^drD@W$d_ENkU) zA_nO%LLf}Lj6lD;kP}l|=1h$%l`sv{t#N!l5l|$K&P9qG$RylKKPN!TWFOHW;&Y{# zC8%;m;e8h+nWs9k1ZQxnn74Qd@_4haG!y8?x8om>Oi0G_df2K1+~*H)nKGgLP^H>4 ziXlfxD8!Sj&*JxA0&{IS9_{H% z7R7q%?YO$0E5OB&+A7Ffl{jvxqL^YnFfYfJaWMjUYK^SL^2-8vMANAh1HvSje@pl{ zrD{YB=3gYxT`2UEXkjMg<(N%xxeGU#8CF+0+1TVe$L z??=ev3{bX+aK3c0iB!6mz)WUj79O)Zutapt)Y})Amz5z=V57*AwQ3aQcpkbPd1PlH z9*5+i62BIZR|x`zNZ+#Paz!bQh&sTXAjGBcxo5tfEx;r~1}2cGBq@A!mIr2iX{~hi zckyo5NlLrZysRoimPWkx)uJee8VpRCnhlPEkOsf+gB_KbFSxV3MwwRa#@($ zYq{r04fT~(fZd5I_msM{N{~Ozs} zOf6y^Lh1fkV0AENwj9ejGak7{QAdT4zVXPKqR63-o}i?M3Svn+b*2#4&|MuQ z_Q$(93-*07&tXmaFxN4_g*7vD+Se(O*^ZlncCm%|4X1hN=Q_6`&VyJ*fP)Z4NF?Z{uTMOWVr7gIpkFVM(84JCWn)5zmB5|wWT#7 zNb}nmMmQ};Z=ppE|GPL!PAlf*_|q>9j51aH(?F&fMs`-*}{$afmRWO11x2t&3 zK*v77e4lkSZ^qVPZHpaN;gH)eg&RvUkZ1{Ktq`L>t1oO?Vxs=n7?Zx1{<+*=+b>BkcIi}3AhrELDin*%>bGBv5QJ7l8gcw%*Wibo7ir?+ zm+H;vRjM<&zlpwTAR6n72BZ?;v=S}qBLUC=Yx$*q*g#&abV%PiI<-)Um5vFQuA0g7 zzqk&}Xv4c*SeVUFQ6t=V8`%m`-G;vDf-MUAZXzL*Glf~vWg>?Onx7YY<0>)M>X4}a zB+U9zZ2{1Q4vdfgjRSKbEqah_|AWD)G-$VIyiszFxE{^3;?hdOWUVk6bv<(FqE>8 z;a`ozvYAquMuH#t6FjwaTT>Ac+(-Z`uS`p5FU$FYZ2Suoz@QCR#rD<*%MJQl1P}F% zBL$Kyxj-(=^?OFyyEG9h!Y}UN-AXUp+|=_yAr=jTD93&k&3^={wpk~XTKLL4Q^Dw! zHEZ2G8U%KxR=7gp{JVK0-(g1(96}?w@~1#np_&_TrAF3h51ajHBgtpD5^ zgPJkrvIDrn=L%Hur&gC@ip}r8Clyk$A<{88|4SeZB&>av`HV5VTE6H7#!9&)td!O4kelSc5MU`VaG`4OY55>g z8R&i4Mp5Yp2#cyvT6kuPSf-9W!okRl=$)P$i^r%t(P>FL$vL1Pl?bB&joJ?rViD>d zpQ5mMpH~RT$iz-0w%qB(VnP{%C={kEe-cXmTP%zEM~VO}|FZLmz}2$^#7bR+ail8N zar+9gB5^(s(S@oCnB^d}o=MrXx2s~0O0={L_U#{IjK)EG?5`1KnM2fE4r*n}W$O(wpHY!Q=)g6p z=_16^LKs7SSzHNV%3w*z_vF*0*s^P}Lco5C|X4Kefai zA~G3S?T5?1(NCR_Rz3WP7(?>}XS2y^q%g}kLKW&-KS~IVmCj1)syp>+xn?sz#C;|P z*w&1`;Wq-T_AvZesq2QGnhX>%pH8mol%n4vkcbiNDzm5hkyBEBka5Z?tLd8kHv;h^ zgNiWr&DFd{td4lhF5DhJ1EE|g0|xNLML!3#EEa3Ug`Hjia3zzEQlB}^UO2>E=s(ko zaRJL-OwmIhe$4}*My!%tl2Nk@Sk!8n3S(t6j0yoJE$z$-TI~S}Yo#Y3J02_iWIm7$a!<@EBh5ERQNX^=FMV56yWfH z|42y0sT^v|_R}8$fK$gpec$O(ULxky7E=USx{W*Lf?6HfO0BwbcJByosUvM+fo_p! zBRMw#a*Z==7JM34;fD?22s4`f->DdkBy8;smVh3LA+|7-GTiuQ1X-4`!3KB#N`RHB z3QJtj`=!_m8%iP44gpccGlvW##hw`B=}oX zK(IgyO?Orwho%9BlL?x7J}B6-R#1jwpu*1UPKDB$E{ zk$+5xHGsm2EZ+HF3NU`4g`skgDId80L>zP~({?hL@jO1Q9PD5XL(Tny;$~Z38uVdn zgj$xdLZc%iRZCMyF2~GJDs5g4V)-jcs)b%i%DhL>I%XnpB%Ku8`~C)yl#iXL_t4_yjcK zSTqjY>)bj{_LQ*20p(@zX4J<_%Otk(uu$77tQNd5;Da@~^~xI9LFQQGLBF%%XyVXlk&z*a&1bfyHU81o$E$ zWB|Q^nXS5bO)8Xn*iw$1Kf@zO3k-r54#*w8B*RQIdZ z0XoQ)AI30iKFI&3pgkg(4nW(COZ?|?bX;~4ltZgGh`wQ&4mBgIG&}ibg_eS`6uCvp;=c>FZV>j)fjy>LHyA#ce|8G#csY8; zO7kCe-#TL@vt<8N9G1~auAM6KX}bW7BK@?bsGZ)8B0228VP`^5Ay%o>3L{V+Q+HVP zhp|u0eQ68xhzLnpk??DU7^wgQsV2MvWVvEK9*(H{3A@3_RblB^NB}**`)Vqj7QwYz z*h^5vhs;&z(sks#f9#QE!yv1@Cx{-sq6mfjy?ZQXDtuhN8qX^NEcv1`bfcR6f9v&B z)KxiH1VV0v+yOP8RoMDQmyy>RW8sy~1MEVDCO!5SbA&)I#@AT1Vg^X<`N47gNVIff zp~o2#dqpG9T_@UWbCW;`YQ;qi?0fZQ46!F7*?w99od#PSSR9hdpX1Q}Y_B244iPMjIx- zy0$^9A_28p;6}l0?o=sk?NdwhVP6wQN|jVg8`0{OM*XJ+T0t@YPfzHV2(bvmL`6Es zWx^~=Y%Pk-JiZ{@axc52itsNnREeQX9Z~wPAdPownGmAQpHrc-brO{E=71o)Ol7@D z*V3H-KJS*-aHlUlMvs$7nWc*=)hMl)SulRw2jEX3$g z!mYk+=k`klS=x;AEz&^IVNfd<(a|%AYa0YcJivI#X&is~M?l6rC@WF=-w*?nP^NRc z*2nm;OvnT_i%*MUgiAZHZQ$P*LM&k?za;4Ie3D9u#;FY>>`?;>l@kPfKlq5flMzN6 zmRF*8Sdng?J*c1ZodI+L7fa00@0N_CDyJhI|5qR*E(^Hoe)pZ@U>Zwm=>S=gB=w;TJy)KLT=^Ow!c3~WK&uiuS9T6jC1YE~hOvK{4c`(&IuTuo zK6}RJmIsL;RV+3C_Y2-F!C(oi+TU?3g%n8Ssqps;u>wtE=P)%Ejs9i&0C8#kTJORT zUjz`l?tQBAo4+(48eGHfj;sC-KtHI7sF}yonynx64`3sOMn;M6WrCuD!-l44sTOD~ zrF*o)dPs#|5qm{JN@u~8(Z3?l8Z5JCyK7Iq#U9}%c3ds7Bm1AcThEM+lW{@Tx$^u2 zUjrLqFmk~WR(~=^&~9nke*6S}j74<}u*^4plM19`o%H5!35gmMd&`g>O8o=>0;7qd z3Xs*+ReH6uvt5XNB@_HDCxI+a#bs4_j5k5~y86GsMv{tr?%B{53?D!X&Y4Rj5sn|o zpHYeaD_F9oL~Dea$D;D;3Cc-;tN^eKD>gf0KUy?b>-bF!Fgpgvi z((jM`p?0g(D)O)09WC@>4H=E*zajwJF&L01Pgf-H0Vz1oRa(Z5I=xzc(Egy3cqk2k zzO8TAJpN!hZ$?apDD=NHqSG_LQk=sMOsq)?iiUa}58?PlST;(uOL;|oSSZUL0kV>? zek~yminaoo{jx=jZE^;}5A11T8=;szP0>aDqWVf1O-PTEkcdW{{+!rbxPeSPLBJ?B z?WB~h?lyU88SXg!iOH_8Q{uP6jR(ak6;u5iQT!D_SfKhV{a!&<*^KUGL05(O&{IH- zEF5g0Bg^$8W5}9gzW1l%i+rGp!SbNLO|KeBw4{d(Jo~7V+i>#84!l}AG+t!JaOK*= zok5Y*`droTO8>=aDK9x?*!4$WPIdhBbkK-frXYecIQ)Mgxkg`%5hC*$b!)-o0ZP;m z+6q!Y|EkX7{y4(GH)z5!|02+ubv9?de3{-YE_SIR#v{S57G`nj=#Tb3{W%IIAy=&J z&ah~h@MB1%(zn-plzdPdU_2-C!qIQKg+jkc?=Y>M31q-L=S%=AuvAHC=Hc|gX8}TL zNWx^&->+ArCL)0}LK2VLpKVd8cV-Qk!%yd=yfNM*2?o{Rw(~(-Mi`_Amgd6!8~$Z| zh-#e@Pt70ix5tnUf@m2qKoM!!po!!bVU}llUq(~9690?d5n3k=NN1km{DZ}SGprGcZLZGR!J)~T|aLLy~2v(mjM zuoaSxLMSoWW#0-5=bY8f0v7cu6hR8LAToVB4xhtTzHsL9dohNw4K3BcpslcKO~VlX z=L)kD(( zz7R63HXuD)Sf4p55Mr}8CrlgYuKQdGMA_NC!8HOtTd$Vjg?+-!PW63Lh)lmB1YUd< z2%N}O=A3&jxtcdP9b1-kfO&qo-i?=}2G1^uZvFX2xHW9uw?>X6F>qgI_+Y$?U|-2) zHri)zun|}5dvwdituc6vbEp*Hq~ayV=+>A3>Ks#gKai+jH?z-eEAN0J_2Vf*+B#WQLb0KU0(zp0R}+e>p}KuxMD) zfvdoQKJg=bT5y)}B5PXb2((OcuUC<_wswml-WA;6D##+F_d;Sr^PlIR78*5KwxhDF zKIE2EY|IM7WgX86vOcMZ**{wA+W;VT{uEh0%lDPHn+J|cgniKdR{~m!Sz0zF7~v}# z2}oL6=`~v!#X|6=6@Bg-vtA^CMh(sRb?2uK7#us1X-}p9nSxoPWhaZyrdY@)q69Q( zmxN8SPYW^9a62Rlj9h=pC?JcT^AMq7`X0i}+t}>tpys(ukP)U7Ll_J@bA%WHQC72y zma!f!<(;5Ln{gQVsrLN@SU^?*SuFA!g;+q{mxw%}u5o7+Q2V`@DwX?x+=Vye7|I?gl2mR2maZc9>9W+sA-G{|eeO1~L4Xdbgkq>fr>DnofWt#$Y*B>q!B!ib0Dn z4Jw0dE09i6*2ziZE5&Jnv0lXO&>W`})It*k1uPuKNZEvJS#3M>Ui=t&nO8D%A*!Fo zSUA`qecN02rQ*T_3@y~3dOrwFPithXFQ;XvJTREBBJ5wwPa$-HFtzVE7R2%xtS+Nn z{KYsRD`2TCL4{TqK+CvnADtxP-s{-r(}x%wqEg|!yQ4K z^p!#^=HlvH77>;RFy_yf9V5oL3f=d%Eea~;w|fct72?yVquolfO?15=OK$39tU!(D z)riHttGWaZgKM290$SG5aj*j>n_Y^AnpfBm8B~+D;5DLa^*CnLt-WDJw4AkJ+DNMy zbw&mOUtMo2lyL}yK8TeS{xqd-nWozySNZ+|K~^(nMu@@Vq1aoFELtIvJ|QToQY6wU zmK9@9=ubGR$T{&iE(!1&Vz@6i6+cW>BBLJul5#L3g4ReY%*J1?P;#QpqG2uz4PK2f z=&r-LC{2oe_X_+`X;zK!$07k%Dwvu1=_LPH6%bWXiGgug<9m~MrH3oWVqcEmHkmhM zt-?IYzrLE6mhz)9m(;mpY6{CM(^m1a9K$Tn?9yS`JnXD*N!^K%f&pNIAht^c&tcW!r;hUe1p&-4yOIiZW-d!R0r7CA@OE%Jv1 zSz3shl$^;$hnXpFGFiub!k5efB3_LwGzGt^-rm*Gun*T?(ZGAu_vzLIzj_XU5dewO zZf5YCVh}+SaYn-85rLKi77%m;OQm5Zc{AiiH#AFC6Ff&8FoC@z^x3VTaj5*YBP&2r z;<6H2sC_4`(y5|4wJX6e<6K?ZEJ=WN|2O5xQr93Hl$Aj*ddkCmh=y1en6zn0Gp+B& zu(T)AT59Lh6@o3VVK_$4e_D{0wW>6_dhdcqVs8l08eud4E~?3I1&3G+lKO+B=uc%C4m0Ed)yIg<=n z1l005sgP~9SY`50$fsen^1?hBFj^?g#vv9SGy#{9dB&&MFEb&@I(h!|CsT=6aK=f5 z9?}YuNlm1fje;x{1SeSl9p`t&-k8M(+NLc`MTYrAscZbRF~m8PFtOV$$oPP+pSCLA z`bwaYB`n->H}X>;MuRYgvE!uYJ`GCW%&}ya9{jn_@M=uK08!4>9@_CqHUCK;@E2CA zu*v-^UabHKC~!GLLH)B}#tGTl*^O@X#Rw|NMu?G#2v&y|p%N3>)O8xcnW*EbOZI8KMm_=4D@vYcTj3c8{ z;p(DN|MM8)6bPqeCi`iEj7!mrRB;Kg06Lo@cx$YGMsE?F^D*y|>(|C{$O|eYze$jl zPU62QScX3b2t{=>488t(y;{*Y(X7k4{7ymf#H!`U<}*uxEk~rsHsZIQ2Ve+g<+{?} zxs(^n5QjvN?fiBj=D!FDr^uhS41f|$Wppm^mn=_x9*@>F-*1ZmA~&8ZUzv&$73E}1 zPt*vp7C{lQI6zMWyZGa)z^xEeE*dZFtHux}@wL=okgXq&&-{%Oiyi`mUi{|+%K~@h`LUe}zF@Z*& z3UrX;{fN~77QMC~ha_)b17^`{3nX-Yd}GQN`UrXj`l5Bh@I`~+bcjZ(9YQQ}b{*rz zZvwEgvuy#HvzW=bzaU0Y9Ld_57Fn}5rvh`Rj4}T)0hSAH4MuI?-;0rOS|{88XFGl@ z_hX?et*gS9?@VD@n$UH1zB0z>w^6-v7G;L8mYzhKFfBsY3A98RIPpCy6-N6O46zd8(RI_dN*q1R7tJ#Cjl0>cCymuuY3i>vagG~;m|xkURcBi zk@7`a;j?}XY)lw7;nQJ3_Huvq>)=)jtc@~GNq-+eqkGuqY&)p>$tyg9SY6GGK*L;r zL1Z8--ZH|7H3MeZ%;9ht-C;mlqSKLvz$F6}*2*`Y@>TmmQDh87>?UKc8mGQ*jIsP& zgfVG0$HxJ=@P<}XEx=M_8Cw@!d=-0{fEbMdu{SL5wj(S^bDiHbh~#whRwd`T8=F%Q zJ3I0?kKjkWm5Mv~l0702BYJiY#^t6m(6spjn3a%42CZ?ix@f3X1$tek->jQ6+62#3 z^mwcjYiQ3<;Hd5Cgf>!dfUHDZT`RRfJ9)CWf>Jm-D{*NI(>rjs5-x@b$^{#3%bCE@ ziwLj?nPkYO##MhzA?jn}}`O8fEFeR33&>c_c~QA1*zg-zOmQ zq~i{fsq(l&=zQv0~LsJAaD(E&m5b z&)+XBZGNRUO9SDReJO>0?x7SUO*EEg{m8>Wk(JNXjY%QpwfbUYz>Ec3z0_6$XcB}h zaUg0$y>|Fu31KZY@R~C`#2Sd#+C^kLyAon?=cl$i$kL`RD z#^?Kog&2uV=0z~_j|jBfu?sMqfL++yA1cqVNb#aT;|v0`O-Y>%J^EY1jRwqG!ge8_ z{Y5Gvbcc~ig%<=yO-%D*RMbL@2N7JeTkiK8^Chqq8fR~>Dw}jDap`fC(W)usskz$;> z1v<)#*e4sIf|6F-xB}e5e()IjTWThtMqjF4Yf5M#suY8Hovih~JSO|&`~yNGuVK?M zarixPVoZK9*Fo=%klX(o{2JldeatxOe=Ed7kJA)95a@RTE%#bxW3QXir-hHvb!6)@!l73OdxC1c21s`J+3AD*nr>rC(A-=TYgGxI0^>%(=3d@`2v* z!J%bFP2H$9Az=uVFJVg@3dZ+^kiBrqB2<$9M39k2Q%R1gXiLYKbWmd`Z4wA}{#s!< zvWvwHnLa!Amm{+k>eusy84VE2*-cjN%LEy%H141?WwJI=U<*l^AF$^?jt%v*36YzG z8JnPz)wSs;sh5NPe488mW5Fr=7oxK)o7n)LL! z{>O0yvIQ-b0A~*c8uiA7ZCIuYygl|?iG>pxzRB~W1lrJ=Q#^^yrh2zTSQbYngr)Lg zOqOxz^Nan@VvNazW?iNJzfoNHVKXDU?m7~e5oJ7^|LAL95)yG3b;qHneE*%(EeD}B z(U#pAywkswbIFYFsTYev^Ivoa?+7uvhPjD0ru%Eo0<=n@Wy#{U^8Kbz#xVBB!5I=R zHR~tyH=dn}LzA0Q&R$fs28olaX=UIl__XvnlYspQf2+b;5$LkjBDi!wnAKRc8eDUMODE`X!eCsyjCD=ke45{jsY|RA!Q>$)0#D-X%(rAjLA6bP^Dv$(zwRS zPG!EoQJ|HbD|E{cEpCakqn$LYvbKh_4F#*G{#N<2bW_Xp>V<9>ZdBp|W)?4h6#*6b z=oF-nTJ`r-Fe5i;CrKFY3WVHP1wOTQ8JN)HoiMA77yJ#tjs%>pR|M?eshA>EH+Upjbb zgVQYntsoPMtF@ZpM+!1(qF2;Z@a{OIv^mwpm+S9~L(*4p?@6AYBFK6R6t)xm8+vEF z8qb(Vjy~3F)HibFlBQD>GxK6l%K>w{a9-s)LCU%jgPUG20f-ngzqC9k*o!W+aL_?E z!F6EqhRdUHMf4mcC^@nPP!w#jF!fbxIah#?B~4b@FH(D@c^C&rU=9i`^N7A6mPw8z zqwZi`1eLW=%Rp`w8%UVa8`dHNOY{mt4rBI z7|i2rU4I}boX4)2u#B)H0;Hb~=My_$YY`)3ilGWhe72CN>9GpH5*J&aas(P-5N=p- zM++V>%cj!Pq%A)VtCM21aKZ-A{7tTAc*CGif&5WDoTik@|H(LZgej>iSp*XnHGwMG zb`0zO>*543a+6QbMJsJ{9bSz%=q{N(`rbn9h%__WaB^K2`D)g5l|EUc#~3yO;)cPg z(6EoTE0j{q?;Y7whKzV-{Fcv45O29xJ6 zzui3Lmj-D(^sM`YMS>yz7}mqqgatoI!8CLUvkW`cz7hT92gA^|kw%R(NTYPrZcPM`d+ zKOSq#F%&j@BE2@=o605K1INWm{I>g2r6F8&;TsW;yyABd{eLMEme{IAAZAoZSXhe0>_!8i~r4%f*saKjJ*vvSKq z3Y(-TPymfY>`!EOSD3rD@@l5}MUIaYr!j!0NitR5r2r_B;Ht{iKxwl~(|xc4){w-sN(P-C@?`|k$;=>CT^eH?;-_ze+IPmWA@ef7FYzx4vNp)lEHwM3 zRXE0f-N)xouSn%JoLQt_FB23sIKpX5QicDyKP1J*nXr4qkvqq%C4m1)|fVv&`0L^TjejRY*}oR`8hvYpv8x> zn#}`wzCsZ7-{-PES6%8W^=4VX?2@)+O^Xq9|H{+@LM#VNFwv4ij1|HxHH<=OT!aJv z8H3sjZ*2XqI5gWeY(wgIrV-jm%^Dn1?-v51`p|&`?eY9dSVRjg-y*~PLl8Y@tDVu( z9aQ?tWXdlpTjZ)-Kc^-I$%HIxXTJAzAnUQ%GJqDuOrFh}32Zb#fnAJ(Iv`fMl3W&i zuu)WLpyg3zWAV7mKdhjJq1{DukNow7Fv~aNFelh@{X#)-5_!{k;f;E=_OGKp43Wqs z{-s&wNfQtT<~8-a(%+bstZF~|LEfYK#fK>b*9f%qkqn>&iR}_bVkl*>tL^{x5U*A` z`gjc0`5(?n1=2>)Ad~*?Q83Fi3|n4Cs|=9kQunAASB33x0ksq{^~{N`oP6^c1*XJ0 z=-j0E%$o`aAvog_W8v-|HJA6IbeSU2&xU^R{n%RtAZsNYs}yF1=fD~Ls4PBZO-05jDP87dkmVN5gf6(jAdnWhY1)GD-zs=G#Kcjd zR#0W(c&w~PiL^8s4%#x7+P2NE{5GEEEW`>@Wa~(*_E^EXKmY zJ-R~wLTe1cNZrD3g;_1-6tXg-4hh=-S#ZQUFcke>+Okw;U=}!Amy>@k(7b8a4gzh- zau6c}Cknz;N(UtUrd8ndqrqTaNSgXOsM(t-Px(vyYixc$Z$S23dRL3N@8Vg2*;-qc{tD=#v;qOAR&O+D3%h zom44aE8p53hhKI00X3?YvvsIKH>tG3(Ans2@-oAWD>=xovzwt2xAH1sk{w!D5>U2m ziPC1iG^b*$lveml^u-cGI$`rHJ=tY(A;!w=x+Z=V{(7OtBsrRLXzQ3C8iOiqb-6_M z`vn>?8G1@8eA^uWRs~YMF?Q4mF)wHYiYxt|T>w_2x#64<@?}sL&UlRvmTUoM45U*o z_5rk9m*g@N5c{<9NNLa+J6yxlQ)`D{fpU%?#e@V&Odr;YGni~8!kSK$+IBtic zMyACfC$qf(MXqlaWTYoW&Xvd(&;)-%uw|OHYi-Lc_E*0L2ES=Kn?Jt9H|X86#;S+R zR{69)0~zg@?qKxtQUUl8Ltrsao#WGjEU#3bt`ivee;`C0beD|%#nHR$aO>d4&Vzin z%uFhaw$PM+02Y;xqg?FcgADHrjJhBly5TreB*DhMbvLPwCl#4*%RhIFaPBX=bp$NI ztFtMu3M)4uW~DH4F}(_?WejUsg>1yi_Xm`eVeH!HU~ztq|GzkSZ5)tJY@9zB zd(6=#&Qj}rAk1hpwz`ZNoBuFII5Jz#g!;MQe7@}{HC49~vT5sO(f2Zm>A=ZBwySib01!@Z{v zXw2W@pwNdKtPHY|5r5j|O)Fy@#rlOj!;RQT1%UPLjYFd?WH<0+gCdHT73m~PD^DCT z=6YTZH!7TYD#_O^lYVv_Qw7s))#uU};@F5Lz5WA1Rxw<*P=IO+6X&0U!R!b!sN72Y zYQ0i^pM8INJ~S%HD}% zDnhlt@;L9-%ta#d%g|iyKl~clh|HVoipw^6Zxf3^Xpd2oQpy7itl{RUx9fck= zfY#?f)*n(eT9edM%gFrk?<|Oww^hjV(i<6*TZiesvC0stbi$tf<2zIdMQjm}QC~ zC!B5iP>@v->K17p{v*W7JdU;90^j$WR4ArOTE+F(3bK6Y_}OH}pc0>#epgFAQ1^vP zD=1xsP$M|H2BvK-yJy0Os1axz6FWW0Vk;CaA{z}kGRw|iMPh}4O;|8uNcn{@%Y)|p zToa65>H&d9x3DFLnUpLw-`57%3XbJl858j11X=Pj(H->-g+i^4V%;@$m&gD&GD(Tc z$)tV}0+UiGb^cFe;>R*1?PnN-{E&8lE&V?O&52Warz90{1yhnRk7(P`FbzHk4xE$2 z(r1iSB8Y^A*?>j0Pk$zLW3_p-qDd#Hg@Sf**akREy2_pPmk4BQtGsod4!*-lG~SM`+fTcCOjD+rCgblgOB@&Rn&Bo3Ow z@H2bjBPu1LC?evX?-)W~!a%G(G&zot#ZBX(HL*7gcE97(8Ot8a zDqNZG6UPR3Ndr>qhYPdV^4O%Jj%>6b8!{%9jl&8<9882Rl;W}#sO>%Y z#ZpE9t!B7w8a_qM7_g|JqXyF+UrO8xtzydp#|71owF=MBRYm<5`;U~mQH%5R?6UG7 z$NAAHFfPn*|0>Xu)>#7VUu65wVhjmfc3UuKGCCC|l@zgkOo?iMJ6LX zm;^5OZSPJ6ga+tb{2(C~P?KC$!eK$sGZH@hE%(`b;aKzz%sXe@&pY&`Mwlhoj&}cX z1R|E9V$#Msvali7d;&7-Vf^Q`jo&EHXesHCH6*cNKrOfQ7@T`&&~Jq?^ONp2!Ynkh zEd#lBk(p1ca-<4o8Vq%QlM=F`m6fv%ni_jaP{g%WS~xSrJh6QM2`=C7jy*AB)&=r! z3bC0bi6TY86k#gt&&lJ9WgDy06IjXTq8nkx_~X^u{V)NRDyRCTrN3K9lq&OM?P|{z zWCeq;lDgGRY!qtole=KW$oI+zW2!x&YHC5YpQbnSp{=CM?oi% z3N!qo-azwEfVC@S!(eWSo`O_^#D`%l!NwC~HDa*crr6IB7}XbgkDv(+n0d7I8_Bqj z%418Nm2)`AuMlF@%0^1;YyUcq>%zD7YB=rFbb{9A&oL^LM5qLR#?KM|OUpUgg&JpV$uyofj)lkM$_i?87A zylUQ%IgpK5YvxQ~r2ZNwoZ_d<8#46|m=zy$iWLm7uxyJ*@)!($s8Rw30kW$Q=dq0I z3!>Q6yo-yp2PWSa#uz5jDyukF=BEj>6p;2<*;n8Dpg=2jh0K~`W#pyUpY+`lCn|mB z)Re!n91><^$r1z|=?<$8pJPD#|ZKL7ZKD zDfj8IN9IzDf8<{Z!_z#GQc5o!g0fTRJu5Dq%;gy059WhV7a?+32EE7|y$HWOZ{9XQRo>#j{EI@c%C9!(s);K@Yd z*+l0RiNv}@VoM_NO2U1a$oT0ExBO&`oKtZ`>u{ zB+~zqocB#4?bRjvIem3~TH?_(_exsg^|b%C{3YIQ^?P5M`%7BJ1%IiTb-|};83UNR z4Ei!HeL%&aZ_?TfxU^!>^z@7Y$P9zd`{%BXr?f3Csl2arY*Fs%AMEPrHl{gO>-3jQ z;F|uO=C))wH|tyc{*d1To%@-7e`feY`aQt8=Izcs-OeR{iCc`D(UrjGI#+`m*wrQb z;)dgf;*x#9anIwgf#<-x(%pcY(%p5Zxa6RMbeE_j3yFkFpW|FwgL4@RoNF`JxvWLb zWiE8C-Ad=$E_UwJM(0jh>RgAd&b8m+Tqj&dexJ6-xz4+tJEPgT)8BB;?{)6Xcbq%> zfOBWMU2qvjq3Rw1k`4-X#Yo z)7<}_>XJ8{-A%;__oIGU?&dGD-0%rm?v^20 z?#ExWaknINZOLMbI(%sBa>8_?f^^%|F&L)o!Cfc}$ zz3Hz0Xu5l(opUP)+Y5X)zc1z4&?nu^<#*FQ=SC6dG~AB!T=EuZlguZs4#YXoxfxy4 zT~p^Y*G##l_riZ=x?56{>Gt3^xt=`Wn)zMBx24qSUVb-ylJ1)CPjj~u*9z+GE}r+| z@^A;_w~%<=Z{wz)mT-r89}j+%=Sf_~`4pGu258juY#Y~%zoxbcx0&~)lyyv}71N69ns zHGSR2b>+P${${DG~txrE!wvnTij{Pg8n$M@N)>l2yoRG!JbnXYDcW}ruNn+!L7 z1ZAwsf>_{NGu_O-S#DOZELS_&xrd2kIqhyB@zfv5at+N{Za&x}yR+PE(t3*DpTbMa z$B;fe2@5Be;g$@}bVCU<3O9{3hw~hRYn+(r@_Ftek4<007Z<0ySNXk!Hok}F?BUM6 zNciRO$rhgL2WGli#5;2VJlmB#H8{5ueELf2X0LONi7fXf?{neJ=?fF?b>e7lp8=0# zxEioobqQC$BjKt@a}RV_LtbaY8-vNy4{^<4$>Z<`{-)O?+>AvD(nz@1h;!!3gqt$Hh;!5U_62b-=>^~PB)(AzR|mGdFZ}X6{6e_@ z!T%gE@#6dZUehk&KH=H)3EwYDa|`&kgmTwrIJb!3f8kpr&lQ_!OA1H2n|L<;8~+SS zJNUgvcwM?Xjk44nPIrGJ%&a5n?suZ)Qs~R~Z@^aZzE^%1(v}aTyV*PVb|HDY0H^kR z8Eu!obQZjj90hGhJNFggW>CJ_@WEli9>BdtJT=qsGm-Xm0dcIO{ZC7G?|?5kE!`c3 zo;AG_Zv9E;meZ#+4o-K6!Isb`o#c5Oca(UR^PYqbHC>%s&+q!)&OJ#PmM;J!?j-5Y z;JM^D@8qem#<}B!Ti-6-H9&{jfuv76tfjs*W+mIHuXFAsewyCSa!vHJ8C~Fu2K?QU z>6+WX?-O`d^1Lp~J>9#LYg&=znpbDJnbhU1QJvuTPOcqwI6cwc&EP#r-PO={JWswJ zp^j%Cb?z~KAD|2mk+1raPOgElod~m0;d(kZT=f9Ge!|mtqo2X`#C65>#%17=y(kmz z--PKv9&4)^^LW+_hj$hy+&SP&PLjWV)C=QmSAByX%|B0rmpZr$`97NXSG>{5&Fb&m zpP+ri;!Z9deC}z^{gQ9*fi2|yK7J>+cA{N(a(l`5j5(d$Kfrq`UHGgw&u+Z;<@r6H zHH(O|2lUt^`jGx7q`9#@{cd}=teHC9LmBb=NKGdl1PiFXw5 zLvgdHr!hPyl8>wSz9fUZoS;rtq`Q3Zo4{rdqn?k^zIdOEzw5!o2h(`3!Zodi&t7-# z0A-q0=-fe`H9Kg7(Cq_ZTNCb9!pvQ2{IrNLBYCeUA4~cD^jyYG-ao>x+WsH-{TtjX zxF+0;p~OWTHMr!`gd0d5&7A1m{d}t>Zx4Cr9;6%%(7GPKkC6YdeA|YbO&ZJTvzHA^ zxP0=yjyx?H&DgLKp2XG2QI;gGabm(1;paKUI|n+F&QRPE=skw#C|o|SrUd`E^%?LU zY4!!Hqa4MATS0mY_&o>L*bX|bBj2>;ByL7C?G1brZU?SuJN*RXbOZfCQy1|v^1&YJ zly=iJ09U~8A!%;LaQa^S&)QGj^(Jk~SOXtTC#`bQ+{?FS+yTW2Pu4?^`a_mS$rw$w z#B*E1Eg72bYWp*W+@t2Kjpo~oyC0sSXC*lr~e}(@u z&*T{V=EFA$@{rAMcrQ83xjy(i4_du|AIT;c^ZRn#XmI+?4I{}A8O{495f za7|m%-HK6J?g+n2_;!qEg1)*P?u34WEoo%lz;kxbOqZqSaOYN%ujwVw1UX=O0{H|w zEe4y#bE}@GAs;j{-tfEWHu8gvS3|ue-+-1p)&D0?Qg?e3ZXteVGhQ_Gb*_T=1CK%b znZ2PGSnV|CBIIX!9bx&M)Yw9QaV}Ws{|`A=O}Gber{L;4Ku_vqy>QE^n`8&lg9m2yfu6&tvvX~1?$7s* zgqw^1#u3o{UHJIxgj+G#xxsvEY0ouB)7=yJS%jNS-j2dIOZZ*0hxSifU&-(3{2oS_ z+BebzkIW>k(Y%YsNzxd>?^)T*sdyIRJo#2%-Mrf51C7%ih^I5|4BTw!wu$G-?Wu_#x1X7zKH+n z`Ml##Y5ZMreb&V_|C{k2Y)zwc`w2Uj-%YPDuD^}^xdVDVn&tivyoUO3I;X2^z97wM z-tLUeK7|D{)JX;{Q7QBMWWieK)SA9sJKT;BTk774!jTk}sv9 z{Ggk5)zq8-z<}TuW)~4=^O?+R7gHAJc z@LtF7hVJed@k!2lnld-={RHnCQ!}XxvXV@8pngs=zvuUo{oUP?>TYfcb=RKnjeWbj z<=Nfba?)E&_*(of=eY#@`=r}Q{IwIodUSW|8F1H=<^sMSqCBS){w3A-M(TTA7k3-@ zddhPL&u75TrTnMidf|HDl3OW1Zh8;sh|E?CUHX8}Y@jW{6ZN#=h9e2LpST}73Ed9U z&y$~a_<7_2VW9b3`j8s@Cgm3M8+U@V{)-b0PNR$s3li?>8oo6X5GP$r^B1&>!{{jZ zUUQUoyHx!m?eiYy;a$;L5Pro8=X&t{yZpX~GUVWT@hmEn;v}YuQ)~!{wu=2 zop8OuBtMM+zk=U`d0s1waHIJ>3^(f-^89|B%RV6Rs%_ zJpt{0Hhv2g_8oKqbNPNC)6G4Of7iw}e@4HMPkEr#j|ejne^ZHXsKTEEE&EgFVAZ&$ zdx)#2bC*z`8h<78FX^A|`tW`wZZUqAk0xGp1ogC~B_q<^O0Y4+HKU}1TfWG-WsTrG z*LT1lmX5^s}#M?j{sL9B5!}tyRCP(86 zaWklgWDmYk{-kKY@BZKoq67J=VSFtk&6<8^^fvzS}&2Y(0G75X>ViMYUuV46F`_j+hjGn~BOMiBN0 z*quD*_Rn%xj7EMa$#N%%yD3ROb77iWhrBb8IBN*g zgZx+XtR`smxX)sV)VAu$1>(;&9mYC6`8KC z5Zx_U9~?iEO<$zDES`OF?~vB|1;|4M;`;KtAMQHbHMmP~Rrp;-etYtKlKgk= zfzD`Dy1RjI=knX*vM+!Z#m>#o@8BMDY3>>7dC}5@d#Ztc8u#Qv+9A)T3mA9Xkhkl2 z-oq23HKU7d+Mea=lu&{>X$SPg#J7zwbs5f0*y2fJYxc7?2l{uL&9^IE7f##GwV9K1+(FahK}xuA&e=|VdWM_<&{f0ND7Ddzx{?k&lFfTFV{akPiMVQ9Ju?1M;t2Dz?a05(3w}&| zGm!%p@jXc!t~m*v6)*X$8Ae`d^Ud!=1D;FqvlxfymTYGHoln0qhxnSwXH$D zF@yeN-co)y631TRSj2NBV+4M0Cq30w@(tW3@ET|WKBKS=dN}HNBY9Aq1JIq|cV40m z{Rg_Huc$xDs(j67Onrwu?8PBoGG0TcE~F2?Nr$$Q@S=4R`A+k+<^p(VK5=wzhd%Wz z*K}7$=K1i&nH}ioPk}$sV=*ymI(RmY_kO3k=C9fY{u)Idk|P*XafdD-Uys5+)YTDw zH|-;y&ydN5J8;r$W?iWnokH_S;x8cWV&qG{XVcc^FQjjI3dh_hJh{B zZ|FNh-$(Ji#L}NPlzPXn+E#zQ&Bb46f5f{m3O^KlaTI>I!XJTt$KYM!TPa-m$Bp9K zKz%RaJ2Vxq%vhRm9h85{tMpIfcLuHp{+eEg9`C2Q)sz|WJ{jb^{;Xl~n<-ZE9o&B5 zltJY=#P6Ppe-H0_q5mt4^~ga<3c~y zgS7Tu>lRS9#3B__c1vN$QQY)0!h` zZUk|U#w9Zby1Hr13&?M8=%+QmUJPM1jOj_n2+?nPGf!wVU1JP^tPk~193lVE zZ#w0YPB47$#qWvGm$t0_K;h@$SADv8tAXEhaT7^5tkHZpkaE!9d_tJ^#LpNd{l;{7 zWyaxaT|4}qhC3J6hcMqJk5xPe>YaQubxvLkmZY3DgRgbdcrWC84bOa@$)4c7u66U~ zLWgP2eFwkuY5z_8s7v%dCHR>)ioA(W$=@dGbQ+laG!jSX_ZM5<#mBYO-zIR?;|`wn zlzsX;gx!yyW*mH20}a$(_wao^ZYyp#Zr*zMpE%y&cW7rm^4}Z(1JS=Jzv2_()A%=| zhTm@wbbT+N{i5s97*>Z|KNo+OQjUc@8_EWR|8p z^3s&%hD7l-wI@GqX{)CZ*E4)yN4SmjKdj1fz4{H@ z6kHq1HQG6L(vTnM_FBT-%`9!c3mahh(@Mo(~4Ll#JCjDuwtyi{jGvT$J&okes0_*DBdxY6`3-!)i zVavbakrl)R9k+vPZEBWez6SF;LHrT@9#cF~emCg1aN^s{?^(y$m+&rnEyh!(A<6xe`43DN_u~GH zo8K3D_CZ%&kmlHql-y6;^C|cL@|*VT=F=W->x|CsEavfNyW}5uDvqo9y=h;%+sNEm zaqPb<&D{;Au=__62kH0W{a__}jn^r=hc-M9Ceb^B*E~yIf0pK+e;vK`J$%RiLiw*v zlm5dk;`j4kr@4b)p|@#DcMINT4H9g_!F0EbFw4oqWBZ)T1KYXVxgwrV@GR$9jZ5Ox zewwyl>vkfrKg9d?Dec`3esBA#y?c3Nd$)z>W8j-{FKub>w(;D=MwAzyMMv>$d$*YP z_1C4lHSqmf+W3|X)-P{jod|p_Zp9Gj3Ju=l_i~=kL&K%@@Jiow_tL<0_Za!y#B<>o z=K8~#^V7zjDntjs^BLT)>Jsjk%{a=aRZT8i!X-hxGw|?$d z3)oKtUoFD#Gsn@ntWUT#Q_!=K?oQmof%M-zpN4myYDai}{}FfarZmU?Ik)>NJG@aNBUZaj)Z+;*uMU zZw|GEzU1o|7<;zd9P0WwzYjTh9Uk~#erKoo#}f_65NFblOrei;ZQPANVEqbrEADpO z9k@l%>hswJ?wQ32cQ^0%;+|VbIe3o6jmMSZmN(F5cuvBt9F}lXd9GgR+%I@e$IZlT z=t&#q`7rJ=+)K>&Hupz=)FO}p)ixUr>^NYlT`_4$rIk+@2`-{61kNo?uiHGlcB(ZMq*=~KJ zz`Z!92pw#Z+c>PqZCX&~IuiH62?=*P&$Dpn;kx0Hv?J-0Li^gaya~5?OqP3T6i?{6VK;kSiR(MK&0RCy z%{*_#Z35p`nCV_7U$^sq2X5y|;>B&p?Z7P|?z_S7#pU3};+Bnu2H@lQ{X&AW_0Dvq z{4U3>J(}g#;2LqOaVtn;W$%PrMLT+7ApHQ(M&8%(TuZy2M3||#U*M+WX6m_vI>04y zTNa^DZ-l>=Cfv*Pt=p^MuW7Uup1XKoM?bOtBy;86j19Prn`oOmXrnw|BHV+-_b~1; z+y?rC$9Znzxf%Bqzh}Z*vydydk++xoqR*!d@4)SxmT?9xIfeVf#?6=-ot%9 zGSmGp&%fYyZamL@#Pd_!-*7Vvy0}^N>0k2uAGmLD|G~BBLBy|V186h2j=0ltXW>?K zWt<+I?pEPmz^%qL4x_zKq`l*6FX-yd!(TVtw{YLVeGhjD?sD9gBlK;@o!fTOxtEVY z|GCf~x0Ake7jE4l^t8AaaT{BD^x0sdH0_{};IFxS6;Iacl9nhBnklxYf89YG@A&;rY3lZUxUw{7k|O3Dp55FAem{l3U*ev_EyHa_IkgG5 zir+8q@8-6l{942B7je4^ySts)-Q5nJ+jnb}3Y3Kc z6*C3}s-XfEsrU1U;C65K)cmo3eCzi+=XvgV&Uv2m_B(I4YTGN}s&KC125usjc_mci zHofksx>Hz{-_xf5g*rjHlM}=-dKbGG(F@fDC|@am9W4%b{3f+myq69M_vjDs2%XcE z-$}l?a-1Qv$c;9iw^;hcEiR6AWIcD{xH76A(XdNCIMHq#4d8uQ= z(>&ItPUz;&a7TG8Cx16qtA7&8KiQH||IkyyPal@|R;xcO>>MMUaVW+_#K!CM`2LgW zb#LW`_#1g)Dt$T{R_ZJ8OQ3#JUYN;mHs+!f^RNJmumqnh|I4^nVGZ`^yH|cAKUDlh zept)B4i)(Yq4zUG!g_j7jWR>_{dip1Om4+?l%X7z*oT&rL&E{`5RTv&(mV7&aDv{o zQvYM=&~S<#Lry)BMKmYct*)tAYHgD=#ib`oYkf-EGSWykB-r9L&xN?(CDDv%?Rc8p zNk;P(?aFfpI*0qd__%(i=g@e?9PjnK&`h?x<^8LVQ%KX>-mw3m9lw4h^nMX9{F!=Q zUdI`n$3u4u z5tGqZF+NNsd-MyZlf8e;H`wUCjepvInaQu_zPbqULUmJtItq#5>MZqPgL-hbd*))W z{C#R@D5cNC0xUxNNOj>@aa4D9q5GkJfia9Mnfp-xW3jq)xjJ>Vx|M9@PR&r~q77Bn zGAt3!GOWNV#P(YMu>ZNRhF)j=Vw`&|eH|Kvo8(UH(tjY=^K0qRzaTdweQQ|QN^VE$ zzIbKC%eyS&F2`WIt!#E5_72tmHRGj!($hXl9a%5k3F%0RZ-cm2`prHZz#$w#d51oE zlW}*l{jU7CTmIWG|B>Ctm47nJog*8C*HkF|-fPP(`OkMxk!iGDQU0q7!ZBf!%6(p!OdPxac>RaTV84um667 zyouXrKRQ(ZOBt{BoDIU=B3xxZOXj#6?+*=4kF9MuZQKB@P3ne>aex&2vvz2>BaC}^ zfJb3?#+o33IrEf>W?xK)9TI3rQg?e(QK5jHG6|EyKbG`4=!$Ub) ziGApOwJ;nYd%kWwC-OH|w5Qv61pOFlgi|Y=_o&0N7D}QguqxSmM$*1)j#-QU7MN{l8bXgwy*{QP@sC zvn$K!<@)u}7}B1P?6c#p#6BEA-#3fGA@T^0VfQz7gcIZ`^nS~_2(qV@?ML?gRDLF7 z>a~hLsI%#5bfQI@vyRN-49+3VFQx50PwzUZ4b#3}q+dpk&5%uKyU?Aou3rA@v%c+F z{>y%2Xt?UW%0I|&-!cA5zk!>$jXUW5a(=i+_I#iHUOg;4pg+PB3?Rm@TG)B7`)=C8 z$X*#nAAtsONlw#7qFx+}__Y{k7)6djdf{_n99fK%^%$M#!bI-L$j;I)m}C54zJ5Xp z`(GHf+Q>Lrr$1AFT-f_Pqe=UVseUsZGcg;3^?#H44s*HNy`K*6Vz8YT=WhDP^{e`} z@sjoz%~!O)uC=-+)uI1UtN$ULRheO-R5eJjBcTpR9#=*Vvtj*aBQ|3zwxbN?s6@MaI^5fNQaMHUCgmSl2hx?*yNTT9P zZ(FdUJ^EAsvYB7@qqjpRS^1S6#^l6{eg@}o9yQ0MU%cX{g^zlA;)3w+3m+HVa~W50 z4etg05Ar5%qy4nAT`iWJglL(dn7g*(FOwa(xk z`2dgb1QkEt5wi7fhXH!tJasU#?eByUWZ$QTgd%bj#$X(Z(euM0VItZ4W#jQ=GUZyWnX0OVQu$_My%2A2f;vJzH z`{;GM%R(Fn=!eiiPa<)+EYu^ae_CdgeO&*f@4pi&%u5_`Z)$>PUw9{UU3%wfJ^Jft zT<#t9@;mwo@8~DI6KZa~6KWs66XN~vggUbR?mHnd%QH&KLIZhBI45ulF~pHXGty{( zzbth0cn-SIeM{Otl*XfFA&17rW%gy4g=Vy%^KUx|JJscO@X!8wSi{g3U=3nPSI zgzAIJft$t4x7)z!{W1^kLJde>&AgA5R zUNZh6%x?a}_-(~0#Bg#*e)#D~?_z8Sr_Sbw8tqlBHY)z1_OV}^b64AQMH_To`>6eF z)Q&bSf6n-tvCs|1_qQ0|A8lNJyzzaqeVX}!S=!+l+TlX&?pWIgkmYW;&;CBh{yxh7=70G5mhj`#e;NMv!gs@wY5#XP zxcb}Shs(bme)RrdhC_?r3de8#Rd{#&{}cYBxSi-SBrO-wwwfza7fo z_)hq1i_Xso`y47qZV7u(VN!GV{%_m6`93H+c9NSb7{w`AxuDX5=H_&4}#Jk3TdY#L0 za<}rzugYS{Q`Uc14|(3cUHe;Vhlf~Zc&M!&9cs+g)FIwKIwU$qhWe(Fp~1XP^5myN zlZ~y7x5kH-566b)A)gMZA%7lPcZ>}cFZs?G+&^(x{!vff@tn%9+N1rg5bn_*p!16F za@}`2?K>emBR}NO?LBAEobX*!zBAcW?YkcLT_5_+^weSB=B?+9o9RcP7D+Uq9?|}a zN1pKn1IYWbHVh+Bgi)v&SrBT+8)KML5b97rtso@4zvRM#(C~Ue7~`ICD8@ug##Btl zOti07|GcICVcU11o2{S0pq-G?{}^tMDVlrKKWN>e{@J1aA=@_aYswE5K|N$s!ffHq zMfy^HNLA;DQhL{gd~Ha6m`7iLT!*lmJY$w;VG+NohxuU%xeT$x`C%2g2Gu?J>aTqD zm%6K_FjW5Pg^+w`T%+GShrUq)k^SHEh37)=m&`?c{WPiTp+Vl-M?2GKxS8}|L+!t z+Wx{2e^?mmHn0n~unTvv7sv+g6@GgJ$8Z9t5JMbEv_EA3_OnsRuGiV$$Sh=kZ(@I= zaWwmPJo}eyxxoIt!nVE6{w3SEt9BHIX5pmKi7aBmtHv36-TQ3VOU40e^1?aphOy$t z?oE*Or`f%Wy|3lo7h0CS7|y#V<+*8Gq*s2RACu2MU2S~8xeeX3#BYxH(R21?)GTNJ zul9_??Bs+nQ^L-82HC)USr}Jw4L5KTw{Zve@bUPs^4x{)x4dIy^#yZR^#86%f3@@r ztL3Eh(^K^Haq0K|tHu|F2f}%T+fi8|VW*;{$f}`uEH9u#X7707ZK(HMLey5;|j{Ga_$``a@# zl(|-pO6&xVmBP=rxP_iO(#jvm=0ov8RAKNNH4PWtW{c@*8K zDbT);e`;gaO?zrnCrY&Mv$XG1hJ~2tl&kM|-BLGUj4&pm@@3_!(fggm{ynR$zF?f2 z?A9;HkXh~=*|?JZi)LZ9pmh%WcM|&-ZJt|od{~$)ys4OunTTby|Cmj$b3Kl^^inkR zh+Drt2DpUs%Dt3fV3C9Xs?xc4^)Byp zAODv7)*5~L>G(i;wf-Q#Lr5+6?H8(-aD@99vL))J3BLUtd0>{c1N=^*LLHgi`ErQS zdtUZTvNuS>^V36;-i)ZdNt2z(;tbB=JgT&*7s=}PtzjguB6cY+TqAFw?yNG5M2E7G zQbzQrqcO(X4F#cQrMAD&b=M->KC(d?3S8%x{r329)9-KN4hGkCN8`;$&98EI^vHu# z*dAo(hsrh?^#P)~C8}eZ7PCRn!ZvB1rtP1k4=_jDKT;onUZcF$if0^kw>-OF97Zea zU?fTgr!@XzsclllS4DS_#_Wwimzc%4x``@=|ov&;z zX8)6IzD4Cv$6McF4pvxw|JfOJWHBa2?q3WG72kO|^!$S|!rj|wJc^9$gXxj`%ktD% zb_#tq=3@6ppVPK_hWmL&4(H9{1p17SBN3z>}RW`OilD0*zEx|IxMi+-_ zhb^t3*A`gb$pm+jY#9AYIN+W`ID%t1fm4Vfj`l+H|6|SnPcZ+FZgx#% z+hh&Y#unY-cx`O2Dyh6~`&zp-{-2|Uqj`ml=HtF5^m6MqJ2j+{=~*Z7rK#= z_UwJ>N8@q%Z@%#-d9HBFCM5-xz`N)=f_4 zHx<)S^`X2>RzGHc(`VD?q7?Bf%BT9Ee)Q8iVsQMAd*>V8JEHtI&wYzf5%TqI^vUU! zKi8l6Dtny10;{kFYq1VB3FSYd{HLPuz5mhP|9J18jLLsB|FPbE8?hN%u^nY7N4tAF zrYU2Sl>ZA)%T(sN@1}g^l(WVi-v4s%f3^2dwl4PmOT7PC-oM}Ve7ZQAo7vg>#p1A! zJb*(uf@3&=Q^-sX1)UBk6`6SQCb1>qj|13bbL z#8zv+FhH+csx8@OEPtUqgTeM|q4w)_?H5_&dhJU02)_>XNFa#@jPRQxjKUZ^9e=lf zsY(BayL~acb-8wawRWED=FVv2v)cCDCU)ytcIyRpEZK6M-Abmo(`4H=ejh#;iiI-~ z6+idg{;~xH#`3S>ShzumsDn0;{kF?FZ#QZDFT= zT^G7HsoRl74vqKaKQup-|K!Wov+`e)G@=bHx9lkq&N{5eMx=WRq`x3+rgtgFo!Cm> zj@%aIcZl*kT=|`l7s~imSuapdRw5?f>?04LI>D~rq+H<;clNXT?en$mGnC^umE*yw?R*!e=plIaW^ zV3&2t^e%QmC(hB&BS+67gKpHkuKbHfT)d*SGW83^YlpsvbT)KE{g1+M(eEzfDk{G7 zLbyiWz)iF(&mGEjC!)1}-O77rwD&*K`$yw@#y`=lu4_RnJ%zOU+O{bFf5rCy2|KLa zIKVfQFJa!nJv_i8Ji!3+7KNQ3Nq^rr@H=ILJ8FBQI}UTT{FD7#%cf+XCfKJ*vVrYAOgLjO4#k*=$(V}iXy0P~ zeTVsXvdeh_-Dla?7ueTiuAluc%qF&UGg@}Df7!h$c6u6ZXkq`(6wYkSMJdwL+=F@a zu1&@kHWY;g^hL-W7WPW>-%HJZvxArLtKwcpu0V`^yoy|dYI=MMyMC6leYw{m>3I#8 zUJUh4v|B<9O;FI?AdcWU@&B*?xGC_X4{$`i&kM7;R|9;I<6y-t~r;{^Q_8s3n$ZPK_&TFDr{zTdnU z;$#xdNTU;3oWVJq$3%^SqGvKWiH>0!0{wF&Kxa{VgUZVlsL^vKEo-`Jz5B*|L0Sm`TpYT$Ca` zUirs7de?B}f6mabfW8PHum5|juQXEm-=+L3Gx6iff0MFtMH#uSY>*8X)K3@e4;e0f z#>1ER{W7dV_PYgP4cYG7cA)bu-v-^@X$Dz(PJPg*Z_%`0`QNSlZ}9zH`*{7Idn!NU z{XDM@7S=kf$3|>MRL-OH_t?z0mAlvb-cFXG9F?eAZVr_^fI~Qf!S$Qt&7acihD+NV zdyVL)@bUV;$G-na<^L`1527}@4)yc|lE<}Ehlhm~QFw?WiDsnHi7eXLu^o@~1@^Ol zcWV>6*HY;u6K`sN zXG#AY_wAB?)FXi;8iae(Z*Job?%@F*;Ry!NF1>^E|DN?PzQ6DL@%(?3Rt(PnXQW@) zZQ|N8T;J#~t4&ex?AyV%Il9-oTu*)z*#u55fY?D?K{?`uQDT=$n^9u{B` zmS7oHU=@0QZ;XWO`8-=Syc*Wh*I_+2Vl#Gsp(Jc2x1+Dom^E3BO6gGZxv2 zF5`pU{G+k^!F7mJ>`U^z6F7x9dcJAj-9HQsyA98u|VE(DO(2Hg_koID^V> zy%zR-+g|@KOtAi2KZD=1^@Zo@7jYS#`Wap5Mh02raMiVIxPhCvjXSu9zCW-(zWr+G zdDR*j?r8ntBk~Cbkhj>kA3Zb-BS)YJqY#_p43`<^Kk0SqusE{cDh>`$oi7fkRX%X4P?x7${*U}@2r5G7$b~{7+jb1uJ`M{jv1w)6J6-O zRLZt14O!$al!nIDrJ-p;X=t8PYOg?PXq{3TQb><24Q-?ORhNdz!kdcen29vMRH`)0 zrgz;E_R-QXmtKn8U11}GZp`CX)l(W4kc$wbFCmwq`o45smo8^Vtl(aSWUX|e9tr%Q zU{Yw2&NZ&BMS4nUSVyi$6wl5b(lx&{Y~8x<{{^!F_3TK4BeRD=Qa{BY(@XBf7$BUo7F?ZcK1~2DD!`qNFX zhM%3aR`>P%5HoN1pL&Le*}m7whx*PR8r!*UKFyx5#MzM{`Mx>BIiCtOZ$2Mt%`?Uw zhE_McFzlN#J^ande;3MxU5*1C|1SJsWFXVhsu-x zL)bHVdZ-vXHS9h(H5_}RH2j_KQ29&k*l&i0eZuJZMrn9=uQC)$!=blkhaYYG_u&Bl zzE78iBa8n-_}h-x%-Q~jaCr6HP#u4XUkzvI=g`pM{-#%rW4@{`dNrKqcM&Z+ zhK9>zwD0jMc@5D%$Q$HMbe&ZPA=9A_N*SN@O{00l+pgWgJ@kD;T^zNA#*(7_mVMtZ zecJy2u{1nzz4ztP@Pv%U{{|xWXY~6Y4h?xr!p=S3Q{{K^!!UYmrSiXYXc$4SW8=h8 zL?4BQNo+9nLqh#fU!uMj&LfO*Z5)a*5tA_$HQJims2$m_9oenj(B37;XZr&i`?c?{ zv#-&DR-_iPui4&hXrI77N9QE=`5gB76!!UO_W5}BdB5LG(l&X{Ow7hyJX?QQO7CJ% zN9zyg(H9`crq1%qq=bb<{Hort4vt)g7<~n~3f1BsUoGA}*79?&MUvjYwy(c$ei}7R z(wA}H1@~Q-J~EnHNIum5v*UK^C+*S)EI+GH_bv9+5A`#GJu1T5i1thRfA8!6_2~bh z`=I`xb}M^W|8KAMU!SjOi~j!({eQBRJEd=)MjKkR*Xx9{8C$U(=^fgC{$=#8CSlil zKD`pr+Py4y=A>tl`}kE285RzZhY-`3IYJ&oHUGhJ#fSR;+@~-&uDD6x@9xua#-7iz z0r>ZRRXY(~M>EpsL>6ao4(AcAeUJ8__PxvA+!n$`ewR`4OZ{8<`x-q(rt!4@&;I|Ob2vAl3ep138i7~|qL0@mOWQPQM7?pC!S(NK{WzPyj;v=7 zC&(ms1G)2-a|Z9e7|O{7!dr$FScNrczry}!_ji(AAF}@+v;UE$=g@ea{clY`^L)1c z>K8*RnOe;LC)2-sKr&_rVuK8NVvy z0Oe#QV)T9F0aP11h_9Be(=UcY+((c+`y#uJ{V%==vWDGX`@Z|;7=K6o6y*X*WjNaZ zd(3Z6;1pumS@_geDbE;(;%;|OhkO5N{LT2gGLl6OjmlZm!#|C`sY6k+t%FbZQZ_`bAh(fGhP?qc-)?8PvVoQj&Uf7<^$ z-~7J5dA+tOp)E~r(Z4_L{G*?Z4%xT0Q_t&Pylfx3-^|2p%tiad=!_%tA7s~lHvKN| zA6ewkc+39ZyY~N*Eo}Z)dg_urd{^xMz2KY)Va&q8C$U( zW$62@cSTlWA9_Dk6b_I*U$@?$jMnlFj<IM`=rW-YI(5 zpWaQ39!Kt^-^sg~>)sV=v^BLU^=?MJOV%gU!`kPh_PK#P;x|b&BaKdEaR%qmu1)Vi z=X>g1bic12M)rbw`Kgn^sxrocSir8WM@6^}V z=ye%m=&9$z4f;(qXup&C4TVM@d`d_AIw4n8&^53tFlWl(2vf3PiaHeA>W+NJ#Pi?X0fgbHC=|m}g9&&T!y&0ad zTRnj4Y36VEFG5uAmypX)RiGb4Cd98EgX4bcfmQq_dmnWdxTMXcrNVYK7d0wf@3&=Q-~pso}cSmk-eXjC&)B9 zk;NID!+Bi9Wz-nQuSFbns7C@xG@wzv(WK63CR@}Wt@LN>kJ{8N?QHyxS^C~3(O8oH z#}NGwbw#uXIHxXX(LP@FoNKs&n@CSk5A(11>iE#5d`Ej5?(n~doU)zemqB-vbHoGm;XMA|z{zrI%0pu-{esgof$X@Sq1X+Yp7=v*r#zag;pMHMD zN9O;(E01vZe$|*KITN!n7tx$)DLD@dun5uKiE92!=+E|VFQaE`UJn1?{o6G&>_0~w zb*Nu$UV6Sc@x|t;$=l*xe!-dk(cyLd&e0dnDh#gw`Gfa+wm5W{&+a6vK+Bp=2OXis6HrN8R^2?95(+xMfwq6 z?!FEDxA5N~{bYmr`y+mH3@303F~pHXGul6t{)gh&Fa2cqU2*f=EOKZZFa5WqpMMM4 z>b?}2=58ZfuG59ni7d|G+4%_-UstD05Vmw?KWpyRwH$YLj%Up9Eb>|Yt2%5gg8WyV zDh}t|cM&oAR^RV3y-r%<;*;1WjhmjPElCcJ;~1ZB*`*w~FGXG>Zy-(op82Mm^e%a- z^R9e?CO$oK)>zU9aIP-`+a=?ZQ(uc-e3*EA72R%=sm9%hrZufL-5U4!Xxe{ z7(m|t>Rh7&eNg47j*KHws9#Dp%+gOq#diwAF!zl>5k_GQ#-SJ;AIgJ0%JySroXl`{ zlR0%kmTa10{m&$Q?-G6R3CgwpZ>zp<+xu*kk@g)5V=|_qYPbFcSxv?;lRg)vhzqyw zminh({Uh8Y8m_R{7iw#kYqL?MPRRZw-*}9E0p|H#IW2PUkcU9 ziTCWK|=3apCU-}8S2>dBsulo9UUmy5$%GP3{IMefh5zoc#VH*bga+#9hO zY4v&vTj^b?r+w4y^fE*?X?B|OfK1{3YDc93v z+csyUEMy0wbEWZlbkj36QAw$bQcJ+t>8tF#<&xg>fjxV1IL#{9WQaIPS_HvLkI! z@h$#q>Fcl_8?hN%@yY(ncJ8vs4;9+n%D)%?&l{)YK7g!o{zK#u97B)#?gZIeGc24U zD?akyb!6Y)izk^xGt%fpG^dv(&)^)+Bdv@_{qKwPt`g;Mmj8{TUq!Cp{fO2GbmJPo zolKGp}$`7h;86#4rqH~{$TpNYh zQMO~v3!%~&LY;abo>BK4R|lbCi#iB3Z0lOY+1$}QdA)h}#8UP@8c^w(k1$4f<4}x= zn2f1tzia*f!#}P6-)~O-u(|o;)-97c?#3DH|2gLWhnW9I>tnV((#8eau9$z{=J|;4 ze^EQlE7FqYvoRN?n1=;egmz;99Y>83oHYJ`ZfgoM#u>5;jX$7~ zjo-x9Z#G8JLbh_Jnv6f74blGlCBj*T6)p2zo3Rz! zQHC+zc{y3*8MTOy6%Oj@i8p*7MB@`5k1xnmEf3ijzJJPheYU>jUmc&iuiPVtO3&Db z12}{ui01c?k&!(go#_{?4>-Yn3NgfyL^E3M7lt(1i7d__y{j;!dJ4lide>6@hJ|_I zJpCdD`wiUD99TD5<2(M73-;<~(C?r@o1NrN=$k}) z0H2-1Q@+@qCf9quH7fM}g}%n~|6Pdo-#y^h_o}|gr^kdx^vbXL&$q7(4NvF;7@YqX z);}Bn<4*3@2N~}_RaPi(7=a>;!WfK0MSDSL_pUp3DC1<;aAjIuK6pMGJvYSnQ4ch! zBbvz;_qD#|`>ghTmb3pB4++J>nTW}lis_h%zAvyTeqI!!x&NNOc+vTi)(E{@6lS|t z9<`f0lzYtOE=Bei+TZ#1>(WzXda;*b{_fz2~C7%j^du3!e(lk08E_f;Y_}1v~qtw`NXvU|* z!8xA}KP-GX{NRn3!&UFGh5fuFdbYN40l5rm?eYq86}k>i~u=gvBtHv_!sjF zyPnR!SJe2wOxM?=YEnViNLG`vY1XIFw_-cWP>xFM!vP$^5gfw_oI+J?K{$0OKb(9& zKm7Df>((ZW4}U$&d1tQ0QF-{&VbArK%x{eiyRoluboiU5mqOCDW*jh=_5=Tc^8E{+ z^54uO!=X0{!;iKUI)_pl?D~~k+?z@Kvc!VbyK;HkR{@qq!|3`uS9|fU{>^A3^ zA+y}MyUr#*njf0bywF$`TCMd@Ax&>X)%g67{na1CFky{A5n?OzL-nluFp6HcTHG*( zJ`N4X?Xf`u^(f}oVmy5!ITdO3&2(}mQXNn0p04H5_;-KU^YcH3ESn<77U)(+GGxtl z>0hXAH(yq_L7&68W8$zrh(1O`hyMM3^FPWrm zGWs}V&T|{}VVbTOpFoT2t^A*@4`}0FB#b4fx}={&RzDmXR*ny6sZLi2?ax1o@@9WOzAS`_?%_aaJlL;)-XEILg4X2&^5}s2Z$SMwU{82|I3m2tdhH_{@)$k6sXwH) z^@kJmuI2rq6Q}4g&(ivM!1@&F7l*Vsw7IWkHfdxhlWwyV=xZIsQ7d9|9_qzCer)9G7u({QxTorGo8$SSvmev zL6}KTD~~DTWV7jA+UU-VIVyT7ax3KXn-Y?N?G+ z&VI|>uiq`eA}ql&tiURCJeCLVD|?HTK{E5YaD^S6J(8t24Oa%0^A_c>d5CzuuMA$Y zHi$c#pO41l)(EHf_x7)l>+nhYVLkUoY{qt!p&Wgm(|01PK4eRf&-VWvpby$7N1ZWF zkM=Ca)dxrD$1phlCS8N+Jm`!3hXGG)Q z?CUAo-$~B0^PBSb?EgFcoIjy3-p+jogX4dH^zQGnuODjLwfkLUH+N>~)3!aDmu+0l zKA*tGp2I#DZtGaKHPYAF=U40>IO^OJ;XS}3JVC6<{(%nr2k6n5e02U#-b(#2H27U~ z_CSJvbQZw~el6lvM25rg{zxVv|lr`Tftf132q@{AJc z7r()E`SU#k(f+^5elr!*F%z>f7p0ho_7DG+^dB@vaM=3C{eRm3C$7!n+CsMWOFx`qx0ou_YCgup|7P^ zeo4LhEq#+I`hSyr^Ag{Ff;@oe{LyvpS&zZ}f8mADtNrNvz`1mv(vI-&`=NPh{pZc} zo?jbNBDbRq<*39yRGrQX2gqvj5P1YK`Z4kZ>P{-(3HiQS`PMEsDErYF8vo7lKlbaX zC@jR0L^IOp#Q)#%zrp=KSHuk;@Bh(vADsW0<~@1F8Jx#OT*g&g!ws~1j~)0I{cHX0 z+%5fo{rslY`u}8$@LFdnn{)L4>21EpPTyl!wegjN{=RhpH-&i{chIM8&wkGNgZBU3 z>T|Kuj4@xMor`vrMQ{rQBx=gZC)_zP~l1}&%VCPzJAP>e#pK) z%)Vy3r_q*rF|@PmJKm80(1mVfkVOtvs|){W|Lb^r)IBq{*Zy~72-E3xN7=py?Gd2Q zMngZ__dXl=F9wE^seGc9wC0NyZ5l z8Yh_lBKwp5&wm+KU=?ChJbwcFpI#>}@eKQ)z77pNe|o-W*Bjqh&u=3(V=K0!4CScA zJ{-Uy9Kqmx_!Zy#y8SNPHLn~0TWV|ub#EG**X|y5QlhQ1l?02=7+VC_zJyIX_ zp>~_zRia&;O^l)xk6AUCMd1c3?JrE^_0Q zKRnydQOd99tMnKQWhh4__C>N?dHSL6Q)v8utnvR1#{adO z_1et@nM4Eoy>XiH|Fg#b^$-5V{yNWScdbMJq4Ry~1MuwJ{_JsM0Ehj?^#h)F2uE-X z(cZk&73cNPyCytcTX2dVLvD&Xkza=Fo+V!VsRD|Gq!y z>yT41J#y;<%p{|>W;WSt{mdR>G}T~g|!)5(W1WEPNuFKKhqy5qnD!+ zozfERW$xB~XXbcs-fIrA{r2CuegM_G^?k@_PW}ja49QLM5$ZR{S8qh+Q$GVITswsr zdMfns$lgEKw~O4~d2{4uV`P6fKBVcL=re{~@keV5e*bdFM((dX7tWC9a2^-&EFWE_ zM|+|xzhphaFJ2DUxNqPlZsQK_p~f6>ZGrvw_R-go^+W8xA8!Bs9Q*Ha)jbdJ2v0D8 zyw&PVj6nN)_TOKz|NfHw_s-t$e(e1HyUyP~>im5)PO|@gn*H~}YO&wGm7emfv}d(h zPtdaal~5#{Q5b`9NbhwIis@09ojtFFiS)_Hy(esb8GhZKGnHS}=~u#ZawcLIUJ0|w zxu~YctEDUPN+{)?ha|mWzjW40r)xFl?`si%Z2lkh^aPS6zMF6T$@RY#-^>dOgs}*F zjIUI_>-@V*`hV{$V?FxU_!sMc7c2kEeb3d(KRLMn`@a4k2G{?ZgI^-NWmth#NWW!` zz#Ha#>0Jkfy;onJz7B)yf4MWeJd0c(`THhS)&gv#S5L84g4~LjwF2A8GSrD<{Iopy zp1epl>`=#%<*rp?9}eIUj^G$h;1ptrBZ+2oA{(L4(DPe!)aLwqfA>m=_T5Kgo4XLr zi=1)Kv;BYP=;v_}m(lkbd5gS;8@L(y3FkI>2lwy*(fa>K$$v|2WAx!}N*B>9b_~it@Q0^ z7|rfNVgfsCim?~AZ!Oz5zMTF0Ci`~>+mamIC-;Un!Eee?j!F#nU-cWK^KbWY58A(M z;LgW?vj5q=nd6_Ze|y;fXhusv`fx|(g5EVr z*r?PG+hdF{XIvsX!86EiaX;mrXbd4n#t}Oy?qoBnjUzleS0K&ZiR7E?`VG>#Njk|v z`@h6}d!--s!W#VlBYFd}esc!ra2^+N8CP))?fcpP(j3igbXAK}M%*^}{utbU@4Yl# zkp3vGXnjxGkak~Nir)}(zrwkR+qi@DSoe%%^U(*-9}vF>^hd}QiW4&E#uI*3^Mx0c z4|;U=Lf-!&ZyN8dUM^kRq-(L?sIP|cPcD=$L}xD~u+u#IuJ^26yKb)ig0nZ>6P@7` zqZiRDeegX$);^z&>|^b-vXdCEeI^^Yd*1!Gq4(3q2wtLp!Pvp4oyqBUV=xZAfA8!M zvd8{{iR5JTea?NqofM|hqqDdZOP`MQHM}W54pAP5%d;yzZ@E6jM}=X!`(|P`vVSNH zbIDT7!{Gl2=r3g7Fy@E$yT;?$70=Eai~b8Wb%k9heQjhTdwFm?p1XxRI^S!7aH9X= zSVUHQ&6)b-GOWPK5A(uLM;oWH-r$t<_8CK6#eWUfVjb3F_ZKFI=q!Sb^wd$$qi?2H z{@i(sKlZ{OzX?BnYm_r8M};F7{!RGn<-ZN(um3h|71lBA**`v19{hCJZGNxf z@YwJ-BVYC(eJ_U-uARaUc8?ARj+*U%N1OXHpO`2NN!OZ@Mklg(_rCve+)xkGzr?Ww!;*>Fyn z=W!92aTV9FGwN$c{mJ)@iE`JBG`?Y6BW}$=-E!j_#tIUfjBji)zOmuOaML|^@bv#j zoH<2*fJbKcaQ5u^sWM97{*ve(Z?WX>?J$I*u`jL7#PQ|>fno^n4E}M>cudb zoQmpQ(sf+AUbl{ddnS_erAr#?mr5sUb~*nManzw62_&2RSH($ZV*1TolwuwhpuEF7 z)VJP=_F2x~EHPG~zUVqEfA5#SQ}Va=E=M+QP~PP8=K0z`?NF?5XQ^+*>!1FA#C2_SG)B2q7~4^X za{P)NUPNt~NcOFvqhq~G(?-c4IAdqEhz z6-D6)+4B$9U63bm3NciC#@aVBiDsnH_mTWYX3_G{8H42D{KNP2*#-F_ngdNq?|J@R zJH%~^v)SmEk(=Y&NN;BO)4Hf?RzX<7{~WI38e#|SDV7qkE8v)Na(-Ct+JBZnuh4d8ZE z2HsExFakvwg~9%bHZuN}b`15}v%&sJ)Q%fNXuS2O^@n8iKLD*C8aqIGrS*r#C)(Nh z9q2?CqO}JZafsF)MDr6>r;UMl&NvifB4WlLsxLYJhh8__vps(*eL5Pnxk)5YuZ{lX z`DZg-&;EqXj@k6NsI(`3&o8Xsr_aLzEW#2jLyh*jc7ta~&%ZYQu);m7um)?f4(qWI zo6){p|8uqeC)xEL`}ckJFS7L9Y4&du`=f*Xv6uaU)=kC+klv#Ixq+WK`mMs*jxv-Z zt^G}*lKw3J?4ut*RHn0t+OTe9*+kKribJj)K}=acMxH=5Jw8LZlIKowSA3w~pgct9 z>er7|4w3zydfl~W|K}O)NloalC#UIOkb~`AzxrBQs`OQoQMhCb&GgD|dcR*&SI|3= zMX&L?Gi1-tjNg!b-%u9Fi@1!dxP}|JiQey<*CKmf%@23Tdw76HsQ8TWb#efCYvoV$ zedoC_f-FMIG<}pWmxNLDaVSQbUkVfHUGFJ>%Kc>eROGm`+!?a_l5xf8{GXp&+hhM< zbpFrB>zOV&tKa=IF&o*Ry&C3{rRaOvT!DF+dGx{b=a+;9^p;KfFl1`1?>pRH5c)E# zK<5zo1Kr3Vi)ejP&F<1rn<`btOVy91>c`TMAd{y{LqmUQSmn2CuommE9viV4ThYGs zHSOPP+P~Mdf3KcZYEIN`Hsv;D|7e;RH@0hB%UFM*ADe zA3D*6Ze)-}4viP2|BCb@I_seIr1aNHzjx7wD(^cjoK9qM2C?C#&R;AI=je4K#f|$s z{URDRh?{sOHi_$y(r}sIRb0aj+{A6%!99FDUR@Bv1MaMTZUvsu2k_74zm&Q9S<0U> zm?Rsx^ZrkH9wShMQ5b`9C`S8-`rGK-tAC4becueS^c)(O8-H7^tdcDsntOk&KaMoL zt%u)ghm#{hyT4PL~iRiz>vdEx&mvjHaOQCA4wFa)u zMNHVG(Te+*Pfm~xIzBgVQel~=ke>3s~ zncz;64Lh74P@g||hVUWJI)Y<3fm4W~Mtfe1_!R9AqB-U0Y}vv2f9`eei6e<-q|u2i z&S22~J;?sW$M&yw{#pC}@%g`Z*}DDgUov%F|KQ4>>|gf!IpLhgMO;R@!#z#b%+tH{ zB|34H{z!h!jb#&w*P#6`o>954x~1NHU7r*&&$!{z@chBIKGDC}CjHzExGkJJxQ7RLgeMq4-nXRxP33=s@=tc{Q2xmbca|KS ze;UjFNAoV}M=MfDqYcr22M!a?2ozxy(vRJ9-O1A4<{Mi8{>Yo{QuV=|a?ZNnr5*Z!T%4Q8(>hmj6<}nTP1?gGJ;L zEW-+{!apnjyOqh~%H}R*6uH!YyZrCmVEmz4`AMk1m)k!p%v6*50c+_M<{{RRox<%p zEd2e#pCNqY5S9J)uJ8WtTVW%)8L@uz5ZFqudtV)J$y`CS|K0p>!wz*DqCLj-r|oBS zPZ`QliM~c-6Th&ghkgLPZyVcSi}bYHhvPi9L;Q~57*606VyLi>Ap7;gFgQLoOL|Jo z(eh8C8H4}-_YLC!^eoQc9M0n+F5@b$;RbG^kG)?}Uugf2e8}Cae{hGqhX;6sXwCN% zasYX6s;d#5EgYROH-bJm|2V;(0QxB89_kw+Gf7^Zrv0C0efcbN+$En_|G(S3A{siZ z(--a-zZ-{Q>>R6{A(~s5$lbou{5}Tf*|)Jt-(s60JJviq8mq1UueJVPSS?M~|8HX3 zB08VH4b|q>v(|o36;_PBJei!1s)RXzGWsuq=syV>5^QoT5TDun>-fbZ`?>MwFO9`0ZaU58CGBw z)?h8xVLjTtr;h#XF|zA8`=6eP(v)KVZ;R|d=|?kK&PxAjdkc_88>$``g^j}5jIG#? z*huRS#ukM#dbB4aj&gb>8VbY>3HL;27!KyQ(Z0uc~Ph@`=X%?BDs?|HayRvURogpGk!D^^Aj0`x-T4LX>0z%`{KTBbubpemo&6|3j3WD<4`IyUb>l8%@3-vh(B|*? zwspwdEg5|oGL>N8SL^fBr(!xfU)MgMTiMJk_HD@AW8?p>&Bk1mVjdP?5&Aw5kMHD% zo?pv{+`TV5=aO83Rak?n8OA`!ia)mwm|Tz86#IWC{v-O2*E-bG6NuKBMQ8O6^PBPuPuB`WYX-J*m!b00PuB;uUo!svzBZ~y`;YE} z+NR^$sKeI3?X~`Gq4p2|?EiP~YyZ%O#vb-NnuoLB(K?p>jx@augXaqrvfuf2p_^X@ z&(8NP7ndWxZ6&!62XH9z!!gv{w|;>=A7|&+T{rLl-Y51ip7hKq<{8{~3NgfyUakF| zpbtRrnlEi=j{K3!NEJE$AtoI>u)NIv#nQI7;_ z{I<5q+}uLz59EpZ)xHgxlt&uKtA6vz^#RwoZ{Q}fUmFtIcbWg*Z~l9?`R}9b|Gnz= zj5z9;(6oI9R% z4;8LIAk(*$Pj$m1`rx^luiH;UA3&1cuu%D0s(c~u|MGs6g<<3f#7>%5AV(ouA5>fG z42F~TK8kN#yc5^$fkeZ7eYJk+_}t53jC;qS7!xrWQ!yPKJB&dhI(xel(b?PG$kC(! zI%_)Y9e5`#i@gJSYPENOHg4yAg_**bjkzerJS@N>EWs!3r)Au$um*en;GMlaUcVqe zWPkax|L++V*79G6_1K8b*oy5aLpdt3579Y@2gpM>f@6`Nxrh_wDa0bTFymyjKQKu~ z=l?ZFZuNRPa(`=B*g5h~?eP?8?Dr1%KU-gtrJq5?cgD*f>Yt+KgmH{&RHJ|j3MfVe1yoSX z4cx#D+`u(sm=5Eb8WM;hffyPX&+g2g*?WHOJ-_z^6won>(E$}yP(c}{puh=CK>;05 znB33W80w_w-v05`^So=n@7nKr-*^3Zp7o2~G1$0(VK0Q6j&CFHxJP)`D1G?C8?G52 zaLf1rve|fm7BYLo_<%9S2cYdm;|q)%=$IoOvTKCv#u)=3jvjH{73Y0i_w79-2QcVc z+CdnKYU2`WP#6c${RjIC9DX5GE+`Jeoi`GrF&0T-RVb#{N_*<%;xK_e3H8D=j?=>H zJfn}-zmxqw71L4P;u)joSBWs&afPuOZ<*6o-eoOt$8(Wyk^graf3#ozM=Y!B)JJJ# z==CSf4|m>tEW~0g#WJkGDiqeAKQ8~H6J7Yd_5Z!^Ml`+1*N2ul+J+N-KT{r$KkZz{ zKZP~oT8pRa^IztlLYz0hUie0AM!9b>w?+Qnpzq3YN?jAzm@1{0p|Ef0PVI<;+7YM_ zR_Xj*$n^{f>48^401t$9bGUeE2%YU^4%0 zHZ6G-*U|qe{=cH)aFc!;cX1yNF@Qm@`~EQ$+vg}BhG-vtV2r%u$_M%}wabo9`=jvF z59PmCH-yH-8=+~)8=-mpw$PIJUdWF6UYIGKBu2U>|9S14XW7s1*k94{SQKLdCgG^^ z)ZEZexh;hE;~sXS%$dITZ1{)7Cqwmk>rI~gBp>>BL+z07@>zX1q{nRvnJwSphxtyZ zzNYn_qqQxulbI}aA|Z4`wS^i}8? zrR_0G8)TZc$d010Mp)&XqOg`+kK}`*u#w!1DthXCk#*y^|xnQg{RF47J;X`bCC z;}2hdyl-A=i}uMy?SgyS1=3VlfAE^}{R8h0(cUU^eZ_b5Co4yG(f6QjoAeKs{$bKj zc8!yMGL8}GAsY^Be{E84IIo#(S)^^VT>6(t{{mrmimcHoKjILMAh*pNJshKVHk#Y_ zn)#OW6nYNwZS6P5@2I(cHQtl3%4|_+B6CR2az2?y6+Jc1d%ZU}oN#;!nX}&OW$*cx z_Z;7SyFNJWfZ7k_Q8KMesKb->KTf;m49?*KF5wEU;yMcB|JnG__gGl}*MqJ^CEt+`fvxQ;HmlpR)fOUtyoV%sxd6J-dK?x`BO)wjJzSbf6Pm z=tdqrxbGRw75_ZUM}^~sX)#8cd~7#cm|_FVf^bd z*Cd5i?Kl3OUfaw5H3okbeGTewxK>#Dp1819So2Qf|G%Ytdfq#wuNSrv*}ZK4-NyeL z>%ZCYR^+GgOW?`)|7v!0Vf}+w-QO$XTrTdH{RiqdvB6zah6?P$9_+&b975Ys&j%gD zJm0~d?>Nsl;rXI4|9hC{`+?_s(epj;`8ImK$klkhuB$vgJRA|vF(i?SQdpI9GxS>5 zr{eYW9O6C2y&2O9_v*I}VR@XuDHPfxGkj0V{nL)m;2g?*GkH{eQ9I;+hHy#P6;vPB z20ttBp;o)Tu>aKzJ~H{Dp1kUu>$r*AxQqLEhyk?uw%g^unD;u-wfC`YUf6$=-oVap zT*CfeF0YY=^||gm&U>w7XQOLtqjp}#&P{3_6M21W^A*tN^OowPsM}qiTeH1F~%VN8g`su)jzf z?OkJ?g_of3pX@6{#_|92$%R;qrC5fFZ)&$#zkdZit_8A+%x;taHkn&RUyJqV(AMrm z*8|_jS?>Zp*yy)P`EWD270GVETvz(++v#*>E5p|K-qdh&+P4GNk{D!@@Co66IfVTxWdBEcF8l{^=U^gSzwS{`5z! zpL0X~f{qW=FDKP6m(?%F)i3CoW8Xb-ecV1xInJO77knSp>WdnrUgd{I-D~PMJQ;^p zomT&4+2`HrKeEno`m*u-+AH~JNOsSq@HbF&A*`j&8`$L;`o5(8EkX$oWliNLc#u@sLe%xV*g*IUq@_H ze*mO<|QAjCCOIBHI7=$%m*~`EU0Bfa5{m_G})p|9d~O|LU0uRzZ%?{J3vy+r;-yg~ce?QTjLwc<=rm}qb z^sT6WplrG)Z||4Kk^h-JM1_~30v|8`b~)aIeet*dCOyIYa{3V*LlP-u(1aX1kjDwc zeE`aT{YHrP_9@4GU;LwRnmmJZxPa!@&9f!1;3}>ow~~z|{3g9H|6;sxWAwXtI{(Le zkN)@j!X9D(u^bq*&UZs5)5?gizY!|7yKnc9y5;-0=KJXNecWIlDm&^i)cL*agz};{ zL*M7LAO7)~Fg%8Ru_%lr``g|^+Lp?!a;Hcn~itSaTx zDpfv}DxXS~Po<$@b!li^R~nkOl!oRFrJ?1uQoi-lkaJ#Zwlq{PDGfC%OO4?u4YjC4 z8X44Mg8w-nZIdGOXD}I4F&)+7tU(I3sN4Id@BdBy_c!_9-wZRIGaDtCi+Pxjg;c`QP8(MjVvq86Wx+dC>eH6RL|HVC+;xkTq9vhvP zH%7f``M;%Wv#_nmyzIH4@SL1mO|MzxJbLXm;huNe^Ugf*e(xyv<*B#6q)u>c87i;~ zd$13`xBd(};opt_+0Xtz%>E~v7qI`A_%@fTpUAlWO!MV!;edD!;Ruc)cWYb7B1!Lj zjelT~_5nSE!upqcwSN!t54@~REv)k3wvZz`kgVAj^5h9r(Nm+eE5_OX*70d%&THFW z*3KBLtxuj2b`BSC2~Wpo&I;j*5fsPgS|kZudo|9ml!Jv7U|l?@n^wWK6|$ z%*1Szplz7`-x2(7csjrOyz4Hj-^m8$KqH#Ac-9*{|JOW!?G8Tz|?zx3WuDHr|`|5VDJcr&bxVP7i^ z>&d>)m4@q{;Zk}nTQP;gxHht0y%_h3sZeIV^?Bv=PV4PBuZrAEZbgzF_wOmCm!Sf? z@MQm3K=vZ zhqfE)e{?)h|D%iEjr?-;zxw@Y`zFEG9jARGZCSQ)-0QIQqVa!?%Eyn{A06UM))@bH zR2=l$N7^PhK`(#bH}QeC$#HEIvhKWde&HQT$0@&`My^JCf;@+~4q;(li1NR%{@1g< z3woS47U$k|<7xU|lYX{&oi;-pmypTI|26u5^Z{KE*CkxRRb0nS+{S-6{^+9b@2Kw& z&1gYkjq$y{KUA{E?~3O>9%2AVwt1C4ph5qq^skmT~TA$r~!Du1yR z(52n-w0}dpXR+V=zMBY3$>={@My|jrtif8W$3|?%R>ZZ;OUW`+FOdIZ`G(pWY2uHF zdlYBr^(eRge}(gQVGs7<01hGl-C?0^o%}yo9#sxJwf`LF9rut87xjOl$$8DIJ^yW< zKbb@866^mPi`Dy=Z2p$$1N_9}@qa_uZ_2+T{^J-@$e;>bO|-+iUeouEC+qVy4;db= z`=6V*jl0OrU?UIaTcCGtaxEUx2hg*G?*ZMbACI*k^d0rj0{aw_Ly_DtJPapCqUvS$ zJ$vd#o!2~Nc$grbNtleO$h|E6 z(m0*osSN0FJd-{fJ&yD8a5vf2Xk4+d%Gb>wC+8tKczBpkE<}}drX1Hw!(zuvk-6sG zJFdH@9ZCM%{CjpZe^%XLZEG@v`mA=lYgS+t)?h8xV@-gw>4 z1pDH3>+1+Rgq|VZv#_WAm#eL@Bm5YWNFgUIyGa`9P4O5VsC--AA*&9W|3p7QFV`=6 zicIZPKHO41JcxNn`9QC~W?Zq~&LIEigG2e(28VO>3%GoK8HM$k|9Ws3N*|7q7>)inhKK%Y-_IuBqT^yrz$E0R`Nl>i z!en~qINuL_Dt$V7dfA4Ll%eRteg8F6_-vG*!tq>k9_FKZxB6NAmO89{x~6`5q<%W9 zenLGO8u<^}`43)YOQYpA_Vp|5YsCF^pYH!NOZ|+({y#6OH^-=-5!e4;=zkVtDV8C* z!TL8#>?=pF)eeYr6IapmKlM$jmowdu`|#_??TfU#U-SL=4x82YJ2^nI+x-6~;~`vA z5uR5*dInSc|3v>kNBe-xBl_~I_evj9s70N$q>)jN*JG_X)?*_!fzy?_KjgTy=>Oj@HaT%_d}@IusOVSaC0ahvNh~nw>cc*fBA`e<%r*oA&Fk|RNs3| z|JovG^B+0e|faIVb6pg;oWUR!~PlnDb(y99jbSZ4z-C-DXT|^^oEh4 zE;};Rw@(e3#23QRZStCHD~}e3V>S9^8^?zC@4XQI;o6^uB)$9>Z1k@^ADWz#LkIFW zfm3KSFRbC>)X=PtvFXO-ki9o4v?M+k%D*!%oc4QRelHs?`mkd9&j>q*x7zeweq~&^ zM8AToxQ?5+jk~yyhZw-1jq)q{wwsSk_WsnEJF?%{!I9)>j72dfU=k){D&qcr)5$*m z|C#Zak9KxEuG9ac9T0uv@&5Xs*LN&@F7jDq|lZ&J|OFV^zC<&U5>k#7$306|18kIDUL=op&2bz z#uwBWU!d>5b-(`q7p0c@Ro1`B( z=QJNI4jp73Ipx3!@)WX-!e3P`;I!j2$j{L(P=<8P@(z?E)vpwXnmEP+@xSV}iTj}Y zc-w#N(EhIr;hgI(;1aIjDz4)u+S1w|+7un@{0~{!f11oY?jajK(EdkLjc=t&`?Fi! za8&yPt^181di{C*7te=|c(F-o<@9!~h2UU-AluVmL-3+W3`=#)Z-Jsx9L}a{r(( zmR^hrcsf7-j{5hJGU1;3*S9~}Z@s@!zma`!Bno?R?ARo~Y%{h;SYiEv_lv?z`fQY7 zF6N>C9et0lYX2j(T>Y*btJ@-eWV*$l)%N)5;4t603$YmauA;D%T!s~Bo8tLDUPFU~bs zi}l!u&De@k+=%V|-`G##Yrb2@&HI&8hDcC&O{v7tB>o*H2Z)1+r|c>?4bHz{n_N4 zC;bOGdI$1RE>Qn!6a3-)zv`FOzY8AM@&9i9fz{816Rtgl(>Q~3xPVKzg0|P#|LAyK z{OH=Een)=wE-v@nHoAJk>neZxtNFf$j$QoOtX#;z4I0I2Nu(pqUW$SGP==)oo|@Kq^?=! zw-rckoG8YXASQ^8W_;AFb&7dF-##*Z)1f`!4%U zi?jSQd=72w`meN$tUBFhto%5G~zq# zT<)Eb-P!|rGLApq?E0#0=1XrH7OvCBI=2`TP`S>0@crh4A2uJHtbft?eBqOka$LLH zn1gH9>2o|CGcg-EVcC1bLkYd}DEs{)`;HZ5tq#rH4>VM?Ws(z|W z8=E1X44RNb^1Su;*%2M|+DCrB=R2gIK>b8{1L@1Y4QyX7|0)B^R6qx`Q_&CoFNPQZ}I7M3GYV!qP~7q@3j8cetmqX9v=DK)p z;x@|lf8HhU<00~27#s%3L7U|<^nF$Pp6va+xdCLQJT;OWjjg8I%cB;bJ1^ahI!=tsGENf#~duA z7v}#zJ2)(+=Y&Nc(^7ipEq%jSMqhy*&n2(C?dAvQA`A5o`?9d_YPWiwom$vmhpZp- zi7?YOtFQ)Zu^t<-8C%iD&Td!Vb_`Sh4p#q;Q~#2A^vvLYSI0J@iA~<@+!pp?*7&y^ zT2VPg|ABbQP=Q@Y3XA*N?V-m#hf>%_KY;rE?Ef0}KR-Yn`SJ4Mkl&Bs7|Ory8~ZE$ zar6`_ex)A%n(vCQ~3xPVKzf~&ZWe*H{0$=hhYWo;hv zJ|1EKao>Y%uXT9-Z_j_XJ^(yJABvur_3Vz}!^qWlZuFa~fez3q|wFV3g)OX!95r_?Ql^@oP=KcGooZ$7F1*DlSHagV50 zvf`&-331)^VsTX((>H;vN*nKIjNT;rWaK|@jy@klE#fzu9_Je->xZdZUp79#Inxo_ zoHNPhqsGsWCCI-b?=FxpH_7*82j5%gI{tpY&BJ^w#3^}fF}W1WumY=4t)8k`^0D^0 zx+^VDXBw3c{D=+8ipIy71n8NrZ=XNSz3i}U}dBpvPBQF0o-{rfB zaKLe&@%@L$BdGX>=l+*-!ZCUhDU_Gn^PFr#4*mKD^56bS=vB{mIPUvv$K(l|!f7<0 zHD;JRhYPrb++Ock-F}7M+3nlFRr+=GY!U~a>>pX(Jtx$dPn;T465?F*x@jdLy{IH) zP`|*ud)It?e9ukCw{aKu(PmzI`{k0*L3Y0Sl5hGY{(bZ7={+yMr2ozQWi*Mq87+6r zUq;US_SRe4j(hECE}jbI>03WD=8~Q~TVnn1k}&9dvHo#yc!oX{^)uWX;u-~Y7%r^& z@JnGNIU2dDm%>=G7}@koq2u+JLZ@{GCODpiynE}h7C_fJ_qqC|Fj?4COvg<0ebYJ% zWN(-gO341-&IxnLd6vi(iYWZuMd`{*a_mB-)_OCdbcCdfZvO)et4y}mu|Bi^~ z7?Ozl0OgjjKV}%;=UeF1Chyqb{`{6foKqb4)_AghO_Q)V|1U>&ASrL=$rGrG*NLZg z4j;bb)5zRWZlKP2X;jxpKT_?^LmfSh%q;H;+m(|~%HK1tKZm@1e=d<%&~`)liw?XOqaUrX2`%ay<6AFlt{|7m-?iKj6C;hzVG>-6$(s9(tbpO=K& zWG{J_?E9iM3X~!DW0-csLo&B*P#7QwZSlQt(oah^uVx33!%^@1Rm%s6BERMcnHP3n`lS9={fUD!;_MhQON zzcJTw{@eO=$wL3P{+#*rg;2by=<<3K!H zQHnC;q%qFXub_8cQ}!O@f2Z$3&n@E@(0#-Bg?q*|2s?m7ID%tHB8C1R6om}g`_JlM zvhUaYB`>NkqCKO(E6-mQ_v!8MTcP}8f8^=O8QMRd?+JSJiKO%qoT8sbT)Qx?VNme@ zZ!*?EShMo|9C-n8UCB%26=dm!b%A_)R~=tR-m~dJVIC@3Esxa1e1clkWqtqhYhit? zH2>GHpA9!%cN=$c9}h8rL0dh4eQ)jP(C^lXr|o-s?2BtSFaKYb|Hq{X#I^j_im&{~&xZBnMr_7bl%fn3h`!D3=>M!a z%j7O$d$14rpFJCPw%9N1Q}#R4_U#L!LoYwnA;0x~!~ACZ3m%~#Lq+&xc#Dsr`S7qX z$#D`XWRPS3WwXOV6TR~RJMfM@-sm0Z8N;^LuiWjNF64z(UbKcEc?!wB!@_Cu460n0 z5>IWnJ>DE&K!#0Q&$g{|UK-W(nw9Kd)S?bg_qU^8a?KT7#dX}oZQR9uv@Njyz#`-P z(TT1N#up%u9yA>PgZ1|uXOTlInr}Q89*QTwJrM@T1IYC{NBE%sNBRe|;W3o{40`B! zbn{zup|Jn%%ihPg>|>xWvf^Kjum4AVKh7PA(J23pK3cLEy}#7oPxk$V=T26>{#=+$ zPQ`T0L~%k8TtNgYGasK>Tay{bs_`>{o^~pxZaSVAJ`xD0i)hR2| z`1_;Jh9g^shQqH94ey=(WcW$r%y4Ma%y97Rm%@*Gr-vV{n;s6Vp02O&rSR^`lJLU~ zC1Kx;{~Gpgm>qtwWp>!UUwL>~d3aR+zyIj}bM58tJ{L;qeb3wX&a*6|_x{rN`&Y_K z`RSPFef0A1@Y8m_oEkm?znA~4DCCaP|J8W>e<@O~Tf;+m_XcH>vZjZflBe$xmQ!{` zUqOemsO+-&hl&4cZUme4XUhB$FNL?pyk!32e+@gA&knn^oBnR%e-8UxcL00ZXN2#M z`Y+*~8UH!#-XZ+xe+fUlI3(nMJ|-NB=QIwpw&ru;-7Wtq>|gnx!lA*>g`a4TAMx8U zB*8rq-qTJ>(R=@bKZ%TU`N`mp{aG}ru(L{s8diEQH6iIx%nBhmlP z@NmlSH7E6{QAhy>sK1<=h`zkhYKj!+l}5AE;(*{dvIt+ z$6@=w=_lxFH#ZD2x6HpIn;+T#jh)PU8T7GFJE~5822VDjk;-r!)@m@uVjOh_mLZKO@FR57^X|0=Pwu;e-ibD}OICLe4K?YZVW|Hcj*%FRu_(p_OhRFQ1?`^H6z!i^wSN|9|9C%{M)p4@ zJ7+4UVGbD9qu|47d|FQ@I$ zsvS{TRrGt~ALolRd95f^oh%9q>9wy))9d_7^kt~eO0%#u>hv2`eAD=fPui!>Z_PXT z{>iw9c-%XD4Sg-vh~I9h4FZcv_I;sqwD(3*owIReJNRn z3UtZGF^|W*9ryhDLw{h)UhNG$8E@1;Z$uNCu}d6#unz}t2yxxOBV?aCA^)xC!!ddi z{r{vbMrP2294g;dhD>?fri*<89m0~k&0j#CUOQM?aDsjs_4Ev$`2Q+C>-qhBTsY&m zbGU#@xPpH3ey@_f)*HM|_I>vg;U;+-cX1!h%9e-Z00wRIjU(5njnS^pi5|zK#6IZZ z^pSYF{?9Aw08}orU%YEa`)w?Wk=&sFTR;BJ3Mf&p5 zjFvn4|B*v0s`cmB?DqfCQXAJaAk&UB?fySo9M<}e_1K8b*osn=p#p8LX;16tuPWBR z>6vQVb(4klU9<@s&ia2ep&2dW%yt_$K(@NBd04SJ#y5z4IDnk6EDq5-M~NFp=*Q6G zz2r9(^RX6(E|huxm9KgaOS}g@_#{35p8iDVR?$<7qziAop>6Ut^NMCjmuFiS*Dye} zyjP<>Pp#Iz-lkrEJ+`lXqo_wYUuQ;KO~|1Gd7Qv0{9%6UIxj!+$K}bM54Hc#YX71M z&GBLfy;KiRz2eKi=~~E$Wa)2K5+l&7e~G3bFk&lpKzc7>l;u z+W+!k2Vy&`tK0k6F87Q=57}^y{eO%7|A75}hy8z1-b7CRY|Zlhf5ti!;_3VR3toY?xdv;o9cP>;H{Fbjt0md{;y9m<4Zh1Q%74crk)NhrFvoYg$aji=*Z-|7Nbi;Z zQIFBC8H-{}z$8q@R7}V3uYawry-D8GwvBs)_3#1yN9$kD6vu3opz@KqFl5yY&tROr zR_ODw5UE+pC*{JEb%Qfo+@HL@*l$a*4CTfS6#P#^e1Eg}Vuh{3TC7L@m(PZco_lW&}d|0?i@6>19kr*Cs)9<3kao%w^*~JfVUsxZX#Y3|9x7M;J2W?kg zVkm}VBt~N_iqZegXTt%}6$y|IfDFO0N}13Z?Wi)Q@*>v)tnx>E{Qi5Ej>c z*hTI^?twKb$pgq9_g&V=|H_9$j*p=gQ+zxrO6J6(xFS*PIPxi=H4KJ$y{oW+bX0pZk*-`9+ zan`>lJH%1`p7Y7d9oorc)!XuaBcDCJ;C^s^D*uoA{nkM9+d~Xs&<@`{`t)NBC41|XyJWxmZzMSy1^fA${C7+LZ9M0j zd>mgj4(QALcHbKwiiJ%;{#|uG`DFb6RQhzxME`G%{Wt!9HoaLHUP9hi{?8@nAva7J zGsJoY^v+&3$r$Yc`eO8)l?QI{kt6!bbMAYR@TFLW6jj!;pvs;_hwJl_-^FyP`~%1{{I@bvVMb)_y6zm`yT8=+&?0Z z1N1{E^gp~~P8Izal1QQY@JH#Zl0Ipvlh(rghV#<*p>#T@2|08ij}thB(`Zwsx5s>j zPQ*P`yXD8H_IG?fyFaqOS1bRwc{gMnqtQw>PvOHC&pBMcCFB-qBTTgZ0KL<5=uppJ zrC&$SQE|8XkH%PsYdZ+5)F!x1-bHeh^#{m@D6Bs)N4ka?C+2uinLMn{slTS3>%8Bb;o-`Q`&pY8plC4R(Y?7z1Mr*Gd;WCyLp}80#DX4%Fydk zhcvdgTdROAT*e<4`*JG&^0$fTFUWsM9&ta_tJS}o)W6&0FS2f({GH|B@77O84tLH- zjK)|LV*(~>Ih>)c>e2p%Yyr4N=CCJ>AMIa=zadqWtHN68WF~HnEt#6wA=B9$G=h z{{K~EpY0-CQ4BtGu{xFjqmEPHPtafRNw@^^`jivx+P^zwi4PG0|A zA}_3a)9(^pTYi_Qel(D%nK+P0O&Lhkl67+i5@|ByxSq^^fltu2h4J@%0XONlaToW| zw&Qn+c61#4U7{0RZ;LzYKN^3R=t=)B(Xf0V(Rl55iKbh>OEmZXF402I(sLJomuN+0 z)$bAy#WR3Gf1~_HQdpHZhSF=tyEpL;r;kMai|$QWdWL(QHINuBtog=3Vk}vV-2Q>Y z1acCx!aEuV5}oY>iOG(qqOdQFGN5bkK%yJfx5N#KgKj^rn{OJ`hx{*f@8m39V`)o=KB%0A8&u5hZIkNRe9RK$yu~FDjjj{zH?ELO$!+PhG+f%gs zAB)09dc{}O-(NSDhQ1Z0D2w69^VL_2s|In5T%mox-}t}7@;_Ox+}P#3J=li>C^uI2 z5P1a0(8mAYjyMOf6J31z-N@5>&>&woDg&BM^7p0nKeqD+B1dnnvX6*3QplhQIdq`^ zSK=Uh^`D&}`@XGhL2lP>-k}dSj)^(rxcPzoj>(E|>G%07ed+Yva%~)3qIbTk++eF* zp5p2<-Sed=riO{3`hUJ=*K5V zqf(hRmaIB&zcq3KlJrUBWYo&EgqW7QWYw`E}Wl`w=+amj|7|$Q= zCw9VA*Tr!w)5)2bjS}>J&AI<#&C~A|sq^haBCIfv`B+bsV~zf)w`&_XWx6*EW-+{!Wyi_dTd17CieF>|3P+s zsC{u>y`?_OlRb_b)V+=3ZaU2VCJXD|sW)RC+ltC@pU|K4>%?YpZAB@PuNeP7!<+_s z?ILyDtDguJ^j#>7Kiscwds!VP{KzX05`SOyAo1SbM~TA=9wZJed6f9c@<)ju55AK) zs4O^8bvN;&m3I>FuDh4mKjU6vU+;~?4|m>A{9wti6ML86NHjK%3k_A{Li5{WLetCU z)+9a`TJDVrd;Hfv96(u(Jf?3ejx{@`-=T8wpb+<@JKAjy;z@0L=Tx%ej;YtHthxC9 zed76$*)+m9g#N_eyx5;OB93FI z=)ISCYsCFT`K+6Xox5))c5S(z_`BXaiMak;%I|v~JW70Dnext@yNTT^?3LeCuiG#m8O zpb5=rS)?BaIkYYi_DWH>DxT}OiQC9+D+*cRcj=wkBEFrXaG(AVJ^PFJAB)&d+DI4> z7Wb|jv{T+hauy#UIUH5OQ+J9&Ek-(ydl_cV7KM7$^%jLRdd>MOe^&qh=Y6yM7h|2* z&z>oa|Np|UQ0%x*`(Xmv`&(mr<<&{?od2e8h@6V)hmr6d@MwAn>9RzFQ(VN?Y-}KHY}wtL%rjS<1|^fm%nAy zb79B0=R(94X?7l(exVo8!hxKa?WeL z_gtu4=ibE=*T&mL?m<%DdzJ8g^xEBg11PjVzOC&{&mfID9B@vv`#(e;K^*^oj7%ci z%P)|X?(W!!uZ}@p-5=-WJstl);-mQw!wKm66s$q{d@-H)z+B z`2VV5;R?Mr`UZ>-xJti{`tj-wb=TAJ|HIUWe7&(gyy^GbxQn=Ve8n5`%CC#UL&pOc zR3VRGD28Js3iC4DV__Z2iQbF!w;r}nBAIpE(e1r@&TUoRGd(}WdmZIHd&k`qp9`bK zF&4#`fJvB)shEybmFMamRG;*mFMF56-|#W9qX%FFxS%^K%#MJbZ6>3>%~meFgKkttNr zccFfVviYDgdatsY+#@XZA?za$AV)t$9zpCw=uoG}b$O3DP9pEP$8lFy8-T37#{Ry= zzd+VLQvZ`r{R_*5yC#DsZd@~&jd>0dqzjJ&Yd3w(j-^n!J2^!M;!zcNNkMjkfg`WLTJa^2HLg|qsF&+Gp_tDpF!erj=C!WCS_b=<^l+(p|; z{qN{NC%Vv$JbKX3t^eD9H0{>cO|~4?w@v08x020cio<>JJj4J7{jK-D#5r?{!!z{G zm-T;dDGo#F!_nh7|C;{qMf$(Vk-{pM7l+a0SR_a2PbDWHj$ufNr*^P$1n+C_@IPh7 zNtfd~GCfNFw=~yCa|*Q|2)`&C8G8Nk;@?~UXOcK8tZnhucde~LpNju*{hwF+aUiS46o+Nx z3dCnI&of;`uRX4h`vdQtjC%~kxy5mg@fyFa#d>VSW|V)%+WurI%20t_*n@pIgd@m* z=dn#vnE$fHT$fN}{GEE{yVjWZd;ha9ge2Kp=lPO-|LVDtO~|1Gd7Qv0oJRlG%@-uk z;R5>JXGfE9{hur3Rb0nS+(z6}=q`C54>5q8u16}jep$k&Hq%p+;Pu~z5`+1 zGqi(e>HnK0|0A_q{@*G8t3T6Z2GK??tp9Y+7(@(reT6aIZxsy=Bk7~j=KZ!S<2%UC z<=)AwAJzZzcn=Em|EtvhY5D)C`k!8ypX+~Gw;2~Ij$%x}BuvIsOh^CUn*Xi+)cf_~ zFw=40*X<=imS8UCq4E{wC%F)du@p&RRm!zx^jdj5^{Rc<=&MlwqO=K1&yZH>F6^(X zZb-3(YuBkC_?ip*>(c92o44kgHCT)F_;~-%M#pW}*xxtQ^AFVX=%UB5@o|hm4|}^| zoch-}O}Es)WXmJx$X*Di7o;@E+Gg*0&pkG2<(?5@H=X6cfPx7*U0^wJYeD#?^TYS5`SlR3O zjk2GPu$b1k=V1orUosa>9*OTdh5Rq9fA8A3j>K&;d(r$?@;-9vzl!a~(|?(7?WDFO zo~*0YGfFhYji5Oe3QXuJ_GW zkCPt<`({xqti1Z!FvfyLYsrdV z#d!zzAEze|8vnP~xJ7y`dpw2B^sR_}sxhr`%wru&g_WTKyRZlQZ~%qt3*9Mn)IV?iMU_!ABb!1 z4|UCOjKpY+MKLB|5(??R_Xp|U%fEphG)QOTHR<6f;t z^jdy_xc>iidY*mpwEsWe>nwf1!sD2qtommrJbeHyOY=l|9@!Vw!NTI$gB%Luf0T2nd(t;7 z_KRvG4VI@6=l>n?AIFeF22IGJ19=qI|C!^N-tug(dB%@CYclV+hisUl{WpVs=bUD; z#qr12|Jg1)j`@rGpZ42pprXP2g@5Ar6n8)0KwUQSlen5ms&%(bIVw-xSc+wcK8(2k-wJx?LG|x`>krb`phvx$f1v)oqy8n= z3ai{?3_ zGQNv?a=S4aJ3jP%oYcPlrtj)|{FLH(tIoHgZ>r6Bj`oYvZ=6RQ>(RB>*beE=@0b3! zrGJ6%Z;9`3x$lpBJpYuR=#KUO(Y$F;*dxAuIDkXQW#yCA=H=2m_dfO^9iu1FGef>o zPIT|!Ln2ec;vSS4vfQ|r#M3G(HTg$7jf)$O+40IKA^l`ER0rI{IkzjB|hJ9`FAA*7v2E{C;fz2Ib;c?DOoLVobm! zlz+io`k$JoPM?bYH~veQPR@+_XV?Q|2`au-B7eOU=F;b3KKkD^cAkvo#bPp!4O~i= z-FV5GOD~1(cU}sW=8Atm@wsqcnwI%(1q%Dm3tvUAHGe#{x+JWjuSNax6783gkT#dA z&TmInemVU8rZ0#0j(;f}&dv&ls%C_rY%=$2%NN7J4b#Gb@l(Q&c1{cLuKPmRzxoU2 z%YHumuxd*9!J^NFz4XRmpAHR!KOLIQOKMv3X}*!qgqG}Q!g~L;5u33Ur6|L(vtKbz zD-nLWX_o%4uY{u?$Opn>AIvd%zG_^N^&yMGKRozkNa_o#Njw{>*XjTIP(KFJXFv70 z|LgpSkh%CN>o!abe=}ub*d>lVs5t(G@YaYap?utnVdtn9!>$*ngujdT?Y9H*oN-?a z-BdiZFtR!sLR^5TJ!PXh{e|dfNy$y51 zC3@dity}PAeb@A>=>N@2;W~K}6+dPNo6p@WZRMXV3b%#b#eL+4%rX8As*cYI!yS)Al0KRoi`qGJ_;Tii^prWF4)vp~ zm4V#r`n&x;0a^MaviVi(0+Le^_Y&$s*KutFbUe^jz;wUOMD-Nw4;gPzSbr$3T*?}M z(9KUo)?eh`P!>Gt-%sFG1>nHzbCm2D-ib|Uq$x5As>?czbXl9$@SQX&8TGS zY$dBU*$;&*Ly}%W?n3P~Wiis1wXg7`|G%1@S);CtW9)0^sFx|Zp9vvC;vcQ9M`7W?;}y0!Ns`~R}A8g}ZG z$Ip2CIOV-KbHd{o`3$}D197Woo9N}n7ZmJ&b$0hG^{=`$?j2q!-V5=&b58Q0^T|A_ zZb(w9>)dl;1zjSU>|GMuQzo%SR82?X?^>ShUU7Bq!o>MrDGdPC} zxP&WcQb-o%dNSp==k)hXPtZRxf0h2Km(~)n}-%PgMVsG!1uXe~+h22 z-W3{N(*{SA^P08CTOP5;$sAhG%U|l{9fD$PRJ^4 z>ZJT!LEnWvNXe_U_`~%P`hISW@=sa6&ssfkp5FEi(t}u@?sHwm&-ewtJ}exfA3?u+ z?hoeT(vwIbgC^wAfjmy2|NXfB<1lNVTmQjv->0l2OP;|wTtM?Z{Y~T*T*YEh?ZKCQtHPh!zjHpae|NBdce8(YvVXI~!vWXa$3qO@c zav1w}$nY?dUOR&QORtFYUyQ-3-{RgjxW@$hSNLe>j72e;v(JSIWY#_8+`}aIJIV26 zbQtU3i7tKnPw&q?R=>^$MM|8th;s_k$epo<$eE zTl(_S_1OO>{dc4v%_rIa$E6=Rw4(BXK40-H#!@Upa)k7cF_wZ}JHx$U6@3lr2fH_< zol`e)`0v$EYyG|+8&R&!mq*1{jZ3p8`c`43C_@EyVGpWz%m4f3f7GhK>XZkuj)-GK z>mS*x*Et7p2uIKpJ{^vcNus|0g4!Cgji&<#Kf-^7L4ybf`B^&`+VTujfJecd!2&Upy;7wtDzhGMx>5J$8+x`K2{Ly27fBsYEm-{{c`J!-_ zypM;7Yj6&bgZ6rsztj%0XTdY{q3AOQW;ogV%jd#Kvj5Y@D3D`Oj0u>8$(V}in2Fga z!CcJ4d_@1>&Z3dV==nw*$29^9>&K`kWBIaJSo1V~ezN=*=Jb)PkbBYdV|%Tk$2Cdf z8W3yg>rq($OB%b6%l~Bc4dbtndSLt&>gZ`?)D88xE}ctVrw=H^{nIwmH)AVG(WVY= zzo?GtRsW-Fue#~5I!gT-_xfpgRsDyiEiv`MU83 zD+h;t^jLSNaDaXYg?)*{o8F-gBaa9>hGu>8Nis{O$P9AVJ`py%|0a58w>bz$jSVn% zuZw@7hs>ioq5hxfKgO#kQHS&t^}n#V&T-DQANOx`IL_lllwV|jdq(Xj?2q_?^m`WF z$kTh!Ag;z~(mzA`he-bz=|_&HUA;{a^I{ z`FHAk18HP@L$R;uzH1(00E700?e9c8mmG@WXuGBTE01?5M>= z{j5#Ga_B&uI~mKqJiT^^=Ql_BNk4^p-$L9wFzzuQ*FdY@J1*25w8sC@aiJDx4m5dKiT?facKL{_<{4r4_r2Wfb6g)l%4 z`l0qSh9Y;%e_%MhbB45yH_m`Q8a*4NO}~1#|LpP~Wv}aNJNiP{{+jP^k@@#EFNCqq zD@GMPwc35TrZA7|*Pgv}$8`d?=KC3N_IM?gx`v2Z{Gg_4SS>(nTe}ZQ3c&>QnVLlcjCoGG_ z^v+?vdn~0dLk~TVZggRVuuAWD6}bjU`dV^5s_3!rxfUB8Z$^e*k06hvt>$c#(&21jmrmu8(UBCh48p z8?i4iMbDt8*Rzq|y6-&p6<5mlP4OCZAdh6b{L?7U(4Uk)r|72<*93@b1E#0RKjax< z=g|MxFN6!^C0xN(T*uS&jp9>u-22Pt!)>zfZ(j&^$- z8vnm34!r??X@7t(*e9T&I1GB%cZ#7Hj*%FR^75h(_ZJ*X&+;$kMkPWqeF7%o>G-jG z(*EfGrR?rwtgi0G->(4%g~L$c2o>s}7%}3naD)_Z@j57`7%^gs5hJD;F=E7@c*PV` zjDrkk7=$BKs6!npR2XN@JncP?d(Y$ETTC%xiYca;Vv6ZF#fTB7aEn*G!nv=t5w5m9 zx4-+xJD<;5`}VY?DY(wp}hW9;1Nn28&3 zBj(^%+=dJL?-m(CwpvyDwMt+_>*?0I{-72eo zmE)UT*dJ*l`;U%=@*mRZMC^a4l}-baXv6vbhcvynlnq}6dKS4#b_W^HsSj_6i~SF8 zl5ZpSKfFV}i+uls^y9k1R9apwU z8^C3)ecJ@5bj`CQvV&qitU2V%WH?*|A+pZ%<3hg8ZJ7>PJ{w+5r> zjjpYMUow_H9?il@;RM;_dI$c%#y0WaUtwc0(J|%L2AD)%g==sfreh`sFW1jX-WYZE zIgYhBzn)`Ue_)RI0ev&Kl5zcwJK}LqYNLKjJHTs2;V$93aWC#ikGfYcx@rnTx492l zb*~)y=oxjhHu3EV^BBg+M@Z8YJ1 z2p+=|cnaB;QQ>K_ZZiKD4QMPm=Tj!8@iF<9ajyMsj(G-aunra2ge};HF8%!7eC-}S zckc@R^(y|gzV$4b<7>C@Z(Dcsf6 zb=A@vkXymG7MEGY*GA0u)yMfovHnM|J?7dS*M=H;gZ6^hhn^PhL6RQ#F*rZZziz4f zmrg^e~lQw`G2$fuXO)A z-K+ckyZv|l$9vLwA0OaDq*uBBqsBbYdl$O@`J=-p^v{s1lJ-{rG0**rKPj%d)jAC1 zm#CdFI-DhkYI*Jk;jFW!R->uaxKUrN~*vgMfkk2ZO|{e=8~SRO~ZTmDbU z`w8zY#r|jXMeeqixHMB_T+44JJ?u%V-+n-ubidypz(Xh>HY{ZE2tDrk^%(gCp2C3o%F|^3J?c2*8mvPF zzS+O9Nq7sk;pbbw7Y1K3|Km@}rr(=iB5vR>qr)z84=OIxuJhusP<8aX;br{ln6(CH z|4Z0;#yX5A{w4fce%vKrZk>8Xc=5pZ!Y`&=5w@voS6UnJrK$fKemUX0_G0_j@IwD3 zVe^zr!j@^5grA=_)=Jvjv1#_D;Xf)b4dur!3+oFn3l&E%3mZ;c8fyL4fORu25C1;y z^6*Oiu#gmPLmEBEV#mQ@;aBESy<~ll%A*&D?f7Nt;_!yw-o&=Wqrxw+dus@93s)CB z8}?M`zw3TByteh(@Sjyb3$^{Fp}uT&sN4Os(75rZp`qZXzS$>3)8e0o=Jij8hmVG~;>Y<0kLqK1G`u6tckv$H#|QWjAK?>xhLfl(G5$b1Km!`J3p5>3 zuh?h&0h)`oyF2Ea{{NT4XEEfL?o)q!7aQ7*o~hbDv>EhG=Kmv$99opYt!P6#^6ek9 zl;b7*e{BQp#TSK<(#o&@Yp%~|dfYoPrEWBq-kXv(#?vPvcS73g5&89(YKDhN;;I*4 z6s{t#L9II1b>wu^jB#I%Z^TUD8<0G}K1eUWjzd4YQP2NpOByo#e|}=q6zzWtwaK9w zH#%nyZpCdVzsVZTLU-E6L78i$Zyo!t}G1=;s^Cz4PM5+)7y|n589V_ z7Pmhevh+9bCgQlJxEA=^^j`NL*JgQ#{x0I46v2xKhFQ3Qg=A>To~(r z#$zHTp?2~b>-Vn-SJ4}5{5P(lUx(&0_rKNsr`G5TS`(&=n~586Bj(^%+=e@lUvEjf zRII1mC44u^^%Z4X%;lulC5->^J{oXg{MiKc^bzbd^6du0jIDF*19%9J;4wUbE&&If`p9@2(#3?U^lOr)2V^RKiVHi(N#3WpW!ONZt*O1qt z{lt(kot%jqF$d{mo?pMUL+HIL`G(8YA?+hr@!MhN#$zs)|C9r9^6R_oRKHXI zjO)8RBCdLoHJ!;PP`kkUCZ9&lJadQK{|)qKumZC8GOp9R*HS!H})skV2rcfR%kwkm@xJ_U2hG_Z%BU;kzws zUraZ~)04_>B9jcN}V+(}OJDz?*m*@8Df@&FB9kj;Za% zh4~-q*s-i_@r*7!pUs|+XPH8p-nrg0^ZR?!c^@C(L&Um4YPNQKdhc@fe}Z*>=$|3C zg#AZmvHM?XjDWam>3vRqiQ3i11du~^xc^f3<=Dn)=5Yv*L{eJKXqx3di{1aBulxUf zuZ?J$C0~}PBa_Y3)Xxv_(Z4RgMmv8j;{2fOPmK+rUl?z{QT=?qGGXUAA3a;9ey)Ds zvPk`ZiTXd;&JXA~t^SWRy|Z7v{7QcDCB}tH>ndD>>o6UI=JU-Y`{jol$N_u#Wxp^+ zfIbJeB97(h;a6l;q;C_>`L8T8=#zf?2xDx---WwzFVaU`1Ma8SdB^q0uW!}uxzO|N zKjy_YkJG2-n1}EP9>WuO3Qyx1bWPy5PqY5OlyiQ3-{Et9d-ec7ejmSmMPX=N#g9k( zJbwEUetU`a2WIi(rxu2Ob0YiiVecO*_zaZSU9gT^z^8_-;;K7<6C_G<&JTTCI`80JyoWe9Acgnoz52m>#*7Xh(8nXUnE%fw&inuT z*AK;2^MgMkKS3@1Gx8*A=nae9*AjlzYeU25;*#UsSF!u#KgV_c>RR1D8V)#apLo}r zAd`sxe}4Yg3H`zN()n@B?S>BPZPJJQ%Ka;6^Xos7z4CmY@;GybeXo-LE9L*q?td=( zi*|G%g*Z>R6YZ{JxO7HhG{zz=F6Fw%)8n2p(f^-FpM;!n78z;x;VN;}p2Icdb*QCJ zCugE&%-8<^UTs#wHzG-obN`wSvJc3Qe=Ff5%kPcya8s51k0iZW9-iZzTX7riz+Jc- z_u_tZCHVi#)IV0Je~^8p>K|lQI7hbZm;cdLFE7@}{~392_c{N*vr_qzQEq=-zC0kk zhmidP|CxLOab8|0gK8}Hy<#JxDm+;iNEqx`Z$ zdu(Vk6`#G-y7uJ2?}mmC$$st7ACVQmQ%||i+V=F%a1x*6OPs}!SJZzo5`#^~|9?8n zejcO4XyG{jZ!9?;6EO*K&D5*NYj7Q=qjs_Uzfjppk82t<;0F4QXr3bf&zArB1WhIK zf7RFaJ;6?7lz;4af}SK}`}cdUDfZ38wzai;FA8&{aVz57&)di@cDx(;b#~ePJ~ltI zl8t8XbK3k{l&7s|Q?|CVtrzUyYVH4sebjeI=Ptx?Zxz?G^FQ&elnf7di_4F5Tg8{2 z$L`?~*K@DacnFUmwt?5+F?zIp4S0h76rxZ6&Gm!p7P_6h1$nDbEfnC^xbUcP{UQ6$F?LBCqCy|>VZQ%_1ux?!Je^mcj>K!4D9cXi0 z+<&;H#(B%#7s|C^_QW*kaW3Hb`^@{ln|&jd?0>g!LVX}V{)cQnpx(b$Kl8)#Oic5^ zq2UekO}veF(6vhbFO|p0UirN5u)KvVJ-3(r&+z}H-?pCrFCTU+k~f#g|753d9RKsK zbjmMQcPHP+`SCv=(9f@*^&$Nuq@joZY&ygSh;~Q@* z8WO$~K8xfb<=PSD+_-q3<}2=$-!Kw`O~b=zaxBJUBC;P}9ww1j;W|u5|JsYfOmg5s z&t07}KKFHKFICQpPpweCFEciuUU6ly{BG|4Y4(8)$@dQ(R))8-4;XwkggK786}RCI z+=aUl_d6+n{PNKMTm4_casJ=H?e_Ad_g}BhPgYCwA#%NOS&xv9q4t<}-*0RJz0ox_ ztSSgk(Vs?h3EvWlMQi}-j_d!|U*0fV|1+AV>X$}xr~c=y*8O+RGgyOlsEG2kd)9v% z_bTnG*Z+^6`TFnCw^;xEJpK0z^xu;$C(QppZT|mWbN$iLt^XhCg#P~;;{(R%-$xtT zwNZ8;rOh(EMF0N6D?-;w;{#S3A23(?=&NL3cCs&Ij*q*^^`yPmE#x+A#|~6{q21wk z`Zu+C)*DyQaM1qr8T->G>|G$uB%0Cgo_9HZ4{FhXwEIrYi_gIKmvV1ti}A>9*8g9o z|KBlvuD?fIb*VV=4b&d8p91+dY6?78_uq(jgx^JSpXa*YbME(?9b4xe)R!6ej7Bse zvC%l?R(qY`J?FfS5AY#A!YB9)C(*^mcC)oTWSk4rM`nbxWX|{9vfJF3gt-PazRzy{ zzq)10^V1)ap9}N3G}aY+pNQkHzZ5=;A-jD0m)b{;EU(j-MvldJOvEJoT;6M+KQvrL z&+k9WR$oI;i%a1;dhb+a1DiXYJ`=eK^6xbH7k%0gZV*?!*?1jt4&vS!x01IZ?xE43 z4&QiOo1^euNFMt-|DIA$p=Zm7hP%bzi~I2a9>ODd4Bu=&ctZFoJcIHd7$;A*PgA!e zHyC$NaVH<0o)(wFCVFr5?N%3rE%a^3W#nCCy5;4C{P&+-5w?pT_}vv@2e}LV#$E1- zagQ2rh&AYBX>OM-fpOc@9FTZqHsHMlTi(d*imMI66H;I+L7wpeTAM$HuCPt!*t;)~;-~2cBPdK-p{gdXm#{aW{ zDWs3He@Fd(!2G;J=I0&M7I5770KR$ZjP`?L>Idl39^XAh{bHK-12VsEKbaNIk)vJ1 z*mymzYdkp-lW-NTLESj_$7VK2yHUDLWI~!r|JmH_KMRdM-P8i2nE8WH}!pKmLB2=Q_(d`VsFJpL9L34w4^d zNM2Zf!7*{1e$y(?zs9>sct_;gWAY}}osplf)*g8MckFLvtb?>3Lf1a||A72Y_8yV{ z=^4l7$9L>h|30GqZwmjz_uM|2|A7?J=tO>8oBH4*(s~TJeSBJ7-Rqp?+<5?9S9-b8Le?ObDu$?d4AieoJw9wW_FemLjSuUq#)nP2$A3LMi*onpizkH_s{S=>9y2LyTRAcO z;^@TC;J3k#uMXQ!3=hBb?p_jC>AmlmIwbtc{UrU~hBW%G{a$$Gh_OXiekc5TvNasl zYj@5PH|=}jSN6r)ar%l-HT#P2a_bf0mjz?ZVf)umsn6`Ch2z67^qXy)@!hbs^}FH4 zssA(leBu8L1I7=O4>L~xa^q2oysuf_SBZDxzp{8E$}8=QN4|r>JB@p}PrJv1_Qex^ z5AWjxl>g*A;X`szd)Hw3*W>?M*!@2l|M##qA-{cs&u|i-<4a@%U*Q&gzw{xm$`2Tc z(HM*In22iipFP;E?0r=py4Ue@_|fecgQboqVcFIb1^#r5uER=C?S_oAcVTJ1Db!~OIJ@DLus zV|W5j;c4ud7uV3TPXvAU0`sZ0`}}9Y55sHbBi5AaS1Y(FG|nptb@OM3`lGXzxidm@ z>5R~{#9X$5>(#%sKe*;N$Dt8T6I>UPMXqz2dGgX-hYD=M7Hq?I>_Ats_7~5&$FaR+ zUx{ZU&&NKV+yU(mi@cvD?vu>FpIM%*h5p_kr6 zuXvu{+NVz6FW!29nbY=vbnd$OY|p+T>&9v)q}N(|v@NEAnnS)DG>YpH&LWvAQmz$+ zCg&tF_J36Wub;>NSfZZ3NV&0Ey&U=WhmHIV=e&ux@ebnNHSdz|;eB*DrhAz>#tQxi z`FHyd^FdnChIVuyg)}##&bc2C;2}JM$M6K6Le~QJc@g`(kbPdxKJ)1_ z%h=}?>@(T2pZ!PM5x(vr{_g?ye--;)%D$5^?rG^fgEd%(^l8T^$1CW)uCE9A_W$q8 z|FQjl1p8aS|0Uy|o7G$SVRMyPsLdF2?Yu3>um3ZjUBx!x?MNvs_m8*f?;y8AT|&Jhv&#LiHZMS2 zZ2y0cd>^&qKOjFuzWslp|HVhbpCCEe|4wxsC9cE&{9XJ1LiK7S(TvZWa}wE4t@T5m z#gIL|e~iRGZU3Jk|DkP`{HMPDx9$IH<=Zm8NnHOTjsYAk&D0_JlkYN?9>@QTC(qB@ z?c)o?zQ-(b*xrjDB*n+_y`EJSn-)kcx2S!=@oZN*y zs70Lr*FYxGhIC|}I^I#=EIq&8%T(Vh{S91Le_(=dc}i^mHNSk;9M?8SJ#|iK5>Bk2 z6Ow4|Hs9DeZ{lsdgKy6Ndsn#2G2Q5ye>4C8X8!-pA%pCKn?r8y&7mb>etOFM^j7oJ zH=CbcWqvx+8_iEgwcq0Wm-nRgK0ZKg;T&yObHa!8#+9yZ`J8k4pFeeB{ei`X9u5YL7t9O0=vtzgG zLmHjvI_mz8yFaq`jQR8Q%#k_ltNHVu;gD*10V6RQWAV8>GM=1>x+R|Pai?+qUoZ|;++DaE_u_s$fWMpnr_P^1ew_|>;vvU8g2(U#V*URq@@YJS zu449=9sO_T|83O%hqkHmKRVc^6w>rg?Ee4^HIiHW)#AUx27AmyWZ@AN%qoeo% zzg1U{=bHbQCEviCcpLAa{QK%0A@asJ;2 zp@?G#rVG!+`T2i0(9h5RyOBNz|9|KIrCswi&db-ojpezOejDzlwxfl220n{!(H~;S;`uX{PkI)~(-_HMgLR`DHgQv)+kv?%zc!peq6u+*=Z*hN{ zb;1?M^7(V<+ozsFb}iR_fu4=pFVM%9W)R2N<>xPsQ9meFKVXB~9oNC;rjVw0E@eOG zv+s+(QGbf{3N|>olAXnE)xPa-*pE%FVGFj!Yk89IP{~K7=g0ruZoMgbZA$;edgCAH zjjpA^F}3stG@p?+5@;%;i;MIB+Q@XI#J!MNd~^Qa8^UkK_)_(jWy*dsKmRZD|2_Zj zZRfm$ckv!#{(qnR03YJQ{69Ybx90z~8lQ*%mHB@kN#_%MhLfmWeQy5W=k)XQZ@#3T z#ox~VD_bFdV@QqXT&UgSC!@kh`e+Qswe{5x`Y$)fPJQ)vx{k9~mF;q&9$;&5iNf-MAO`BaYvCfP4s#;4z#Z|Mvv_{P@48=+EHq z$Nv?D;o_gh8mvQp{day^1^xW`@A>vW{Xfm)_@+qk>zc6E8vA9|*;}{U`u{bf!zSmX zcJnXIwckSDhVAH?XMBM1$bDqSIssWSXIyn1zqFoz+OU-WTgv}kChgRBLNfE6&`j=d z?k?;>EgFzS8`9`9&bfP)@dw5o^pbtX9c0ExA33xf_>T5PeY!_nAK4+Ca$I_)_5V?Q z%(>FZ;tjlsT0U~kaciy7&#yo54*gv;Ynw>&)6cIz@SeDD=I{4~Kfs5G^Y7!ln~&%l z{-o`Wo%w|R8BSum@40S;``@SzS7jX6R&BGsnS^gAd0P7)%3lrPbH{#(vlvp#USK3f zQb#}b6mS}_rm^f z^v*^44<0m^!1@evZhx$Qq@@|h{gkP*#J-Cj;n;?sCEtj;y~1MsWAJyz%?Rh$A7KCA zp|>k*-X&9|#!s(Or>4J;574tv`HVjPLVkP-nLEzM@!OkdXf^&D`EhZ@>|KP1` zYzF9#<_MK?khk4-o14e#QFE?2I^B~DRnE%lxFR2>;H?tf!?S-5!e4Of7Muc z;pW(PpiYsHev|%*IpX5JLizRoFZ2FV_CpeP8&VnfK3}`RBK2?KyO5pjxub8M_dwRU zu6lKb{5m}g+4PlcJCfbu^i-MuuWzPL-+l(!QvLs{^#AMcZatv?oNPa$ z|DQ|=r^!yowV#?49+S=!cnVJ=UF;Y3SZT;wL=RZn^{rHp zbt%thy>vE9hivj}5}r%)jC5Rki}=<`_lNcx_m^?)-*~_I?_;6w192^;zb${;r?{Vs z^+EZsJ@J2$o-B=7{*&Ij()};@fAluQF@+cQKUnHHimN7j$Si7?O$u+2Z=%Mz`E?Pb z^|tUk$nSq}P(8!{#{Cca@0l3h6+iHkk>Ne^eSClq@ew{j)*d*Yktgx<8P|m`$-&?J z-{CCzi>1@Uw&H&aTMu6sUYvhj_~l{!Wvj-9$|C(`d#8tA73)u%J3ds+A0J*W`fm8O zYuYvDdts-3yH{4LJM6YMM1z&pBUxwC#FN_`eJT&)z9+Cxr4T&iyL(}n-p|RjEp+TLl{*bYqM@H+5vA@;+ zk?M0J^~qcus{8p>C+wYe)YzYT-&Dc>2>*`Z^pU8T@o!2M-s{e7x_^_=+|Kr^6h4QxwLwUF;Y+tC)5HBtKUZ|WsG3+Rv zWFDM$O=*usnX+Q-qJr?sV(p5)hwNK?67dr;30L77T!+EmPYTn?{;TZ~K@L1>d=Ysg z=HOQJE>}NWEPvDQK(1Jsh0FxDex)+!xHX-{-;H~5KhmxzCH?_=b&>slyvK*=kKqy2 zvhOwGpP)Bx)Q8n=Ttw{ue^&kfj5!*JYs55dEeva0?H9<8UZ?Fnj_-TQIjP0k_h(pp zhyD!Kpob0ZMISQABFC22v6uDT?B6Q(e?6PJlWk?kliKK;CrH=171)F=*oN)cfnE64 z{DMO17fauNXOTmTK8aSet=89o4*ECeCsg~dJ<_Q~1CmiM<$sS;XP`GGTpQB#9>hHX zlEQI}WD~OD-oTr98yD;Y+xL#}yLcb@{fG4{eL(*ZAK??!Z8rX=O8x(^`hU!qC)EED z*O-XwPsjTIXO26G&+#SBVn_pjz^s^f&`hz516 z#zo%aeDf%gJmS6Wi~TF!;~{e!UE^qs#du7_BwU4S(0-@cHf7$8yPB;-#6L2j{eQ{NB-gd!>#K7 zXhvLr?=ETGjeBuF9>7C*1YI-OfAk=>wg27xzk}-kXhXc$jxnA;(&$9LzWv9fGZ5P< zZuw4lf*$)HpNes>>l-7V!5R$e2l!O`%etsPYzzUp30trY?cL^Hk*Rw1e|dZdeHZp1 zmJz+^bN?CS$ER1R|E)4_!tb?cKr-sBMWJSyeq4HE<+;B9>}%?3!g24kBogcS|M=GY zzh!)B$E3(KKRip%&;Pqr9gIKS%ir%)HfQ+ZS^jvAtXs_g)ppQ;Ml>OTB%1MtG~UG9 zcn9y|J-m+(&{e?yM-O_7rC%idS<)|w>HGd&ODliBjj!Lnfd9XU|G&ib(L3kz|EHQm zAf1o!2|h#ZWd6VSlk`UZeuMUm&*@(xjva_=118A0hum3lLmK5pjKtv8MPW2K7UMC% zPn$^gUuJK2a_uxw=T zH{eG6!})(@@;`EDSuFpf?YQz;`S5r1|JdI-(zzA4;SQvIN2ykJgWkK7eZk%Id-1pP z|JHwT%Gfh&(E05F#JMLAk&mF}l>B#?Uya9vpTOVF|LZ!Y{6f!isRqDN{tgFdf(tEYD_N=n58~q*RV%q>R>HvM{fATxwUGW25=BT`G{=i$td0+%$fuC~b3u5TtoW00N&Nn2=xeYp#$gi%TZV-#WbD7)Mh^UBSlCYP zz%J}TUDen2zsUP2l>dvpFEYRX#4Pszd1Hzk*MKD2kVX%(=-Mm)pa;F^Lk3yokYE3Q ziu^zM>+yf|IJWh?{dq$gZ=!uOKbws6e%~SAMOxf@D<|@BkgY;J8 zlF8{vIi`*+tVhFQ*M|JMWUE~BO7?$+dwTRt$iDVpVW#76z>S!LTX7q@PFw!~`F5W9 z^7ImUd#*f=Tq*y59Q$9){*&$d+5f}JR;1~j7`T%EuMQCRHygNQ__xLn-XZ)9TWFj zN}{gxY^X0g>$kI^5l#H81d`ony}PsFUg&+rlcvc-%y5g!ANssJ7?E=4(Z=hzL^X9uRlwUhEyeTd@+kK(Qaf!KS&2Lryv%%}o z?z-M~Ov-hp51kF~&?_EfchMt%_R5!i@@Iz33dj8&-*wD;cpo3&L-fCXCVWH=yzo`{ zg!~LAG1&1{_?-L_XE7ux|4;S&(NN+#XSpu-l348dyS`@EKin}RF&bkr9uqMMSK(Xp zkJ+6I?SB(o-#FjjKK9=|wvp{e)c?u+{-R{(>Vj~MG_J#RRCk{VGs&8RXF~0vGok!< zL&IF*tT8yT4Y0A$x8qw*kV)ZYvTVM5K35ye2>$F6DP@L{4?8J z#6}}gz(yDIBgv$5TKbI-LL1^-z7BqQ${3&g`rsw@_9-;Jzg97ULsj#u9QVmZ9#1`oU@cUGEyu)a@FOq&L_2&$&awa>uQ}O02^9@dvBv=g04r z(#w#WuS{@GtV8$7KkLP9#Aa;8;0r@TC0T`?7+hj9e`Nc%?P8h}UX zy(PX~9HSpcuAgo2U+339%#Z(H$j3$F0Y2>^zU>h{F4?@F?wk`ih0{2Lg66Qc5F_xd z_5aif^8R-L8;$(>e|-B^{CDgBRUaN6ilj3J<1hhn4Zs?dzv7$Yi#H&)9~b+rnaxgi ztAnUJG?8n4fAJe!w^;p9XX`@r_^o%U>zl3afGj;XSskfr zc$n(AX_$dosMR-7qb*}Lz4563cT5R=E}D;fHsTVeJS+77urSOMKXCQ1FrQq2g;<2e zSc0WkhUHj+l^Fc%u&|1(JFWd=g#Q@h`A?JPY-vyNAM|E&wPQ+AhV?k#PZs@$jlx|= z`JczVW3u;*{7+_tvt(|u{O>uo>PKlqJ30{eWsZ9ccB1Ye`^O%|yxAyE#k`pylW3kL zpFKPzZ1q2tsKQR{Mh)uGHAViM=J}Gn@@pTNU!O~!%b{g~{5@Iz7T@mo4l?EVG}-CD z+SeO{^OiD2T5%3wE14FTssug1uh&N9$i@2WUy2Uq`^f!hOeh2Py0_iu%7A7vmH`L+b`Xbf7=v#O4M)g+9K}G#&~Pk<)xVFE zajaF@D&Jda2q%P3;WT33UUnGUP8PJtABb}RD{d(Y8>}bTd&)cVelj)QQMY#^-)89} z99M)fDA)cmj+}sE4F0wtOeXukKRiq!W8X;h|EJQYVFqTQT3voNS%NtCZ!S3xHS~s+ z>XoYt!hGQc$orM-LX)~>VygZpW5Mf<9d982ZvDT?(cz4HTj;z+Sd1kw4=|KrExPpM zck`w5{rijg-}Cw3$nUeq=WaP+{P}6)&-u#j-R1|>pBvks-f8}SDgR#^K>ZQ^{|Np+ znkMl73;F;2_-1mc|67LT$Qqluf?SCWKl4rK2kjc?|BLy|42_9@$aGbu8CK>N9K^5g$5_3iEFAE58B=YuRg=eYBB zxqgJYaJTe}-@?SRQLB#oean5gC z`#-J$8q0#t+48>l)LdnO@EPgUZZ%%LYG^2E^(~WyVsZ&KsQXk5 zv!+uV!#8Gtx&vw^}5#iVF_mv`RKOP?uc9@&_m8J-zj}^|EIpX|113B#HsM&nbTqG@zY^l@qg+6`ZARGueqog_j%Yb`}0sf z;>)o9#9zavy?+V+p>J)T-{xaW|ETcuaihZvvrdN11=3554!FhRPEo!;XW)!>^Pzi~P12WyM2f)}i{pv=f|?p9?OLpT~##&ErEInk%R4&-=bU zps}H;YHX;knH2V5x3-8Q%8LJ-FwxkPiJ``Raj)&XA}o>SQY^!AtiVdF!fLc1y)Lw& zHSuqu<>>TKDy|Hvf~&Q~Ul;0>U-h$;DJ8CjY?`L5NhxzO>ObUq$8N-CY{iB0qdSC3 z;jRVz|0VqY724NF@ChcI>wC-6a|Omf9#HljQU)GT7Lpypac#cXKHo`JNu&I&;;@^H z>-g7@^%%U}_jaGPKjZiSL$TJYJ|gCVQ2;J>)*W?Z-6i!~q<{AsogL^rJ3eeSnm8+|f8i|96T1E!H2c@3?uD z{_i6F-_!Je`>kDHct@qa>z(?g$xh*}l<@%>{nKQx_`db}?iX1f!2JFk**)=yt+7?Sa}tN$-vP(%n-q$iHwO9f&yN>` zar6l&#$<7yHmD+%U~^omCvBNNXyR z!p)VJ`ws2HFr`@gjy@WDG)(i`4AhP(4ztMFs4*#}ebJ;)LZ6Fyn2+=X&kqaey=2eq zNns&<5pw;mjUCFkw!VT%VX?R+Sc+=457h;s4K4 zSD40sU3Tud$1#xQfluSu)!6W)`q@{*LK%HMHexfjq7qfui8ywBH(7&vB+!Z!2LEX7 z2QrS~%a8+?7lgg!KJ3Q<^se$fFIVS~Uk(cAlr33g&?jHSJaZo(p9zNGk@Ei5-W5OqJ3SIT`KQ6TYEq1MJ zYZm$TzXRXs|8rc2I#vp4dgpF?4qa=k!w-ym`l6&G4|y zam!I7zF`@^dWCY6Z?)kT`CqtMI6u#xT;aIEXA8qhaursi6lGYCjo6H>s6-V89xV(z z$$srP?FWq|q1T`u38dX;3a#|sde3IJFufZ&_C6~vgFervuKVlp4-;G`^6T=Ic>c3o zr+dse=bPh;_X_XBejG%Xx@0%<>&wX37wVVFrQ9Or&lJ}&&GRSYno1q?RI%qj&hz(8 zSFH71{yZccl2)~GABV{r<3MY1gx-&%XgF?s=uu-n;@A%|iRMGru~UaX=J)d7$W!x% zhU4_umvDkSg*etOhd9}MyZdCs-Q}3r&`&LJgMHqwT&E6RjXhPnP zPYeyEzRPibpMYYl? za%Yqcr<4(e-W4{B+loq*-!dvxkvp*)gYtR}*?+Zfgd8xJtKvoP>#I?rRXByUe7`vE zZ5{uw`iO5qT#fW={ZEFz7yA&`{%AzgM)iLrU0<{7Jm4La_3_n=7r{4T@^kw+I zHGE+8#vEB!#{WkH8qtKrX8!;BbL0PyxQ2cl#W5Vm37o=dbnWE-C;0z0{C~dvxAq^@ z4$*Ru|Bp7bqa(%t&+z}rPUp4rug^%QAgzqW2;|ov6kkN|RoCso82UKmR=f6yEk)dUCG!hNgwy zBT8K3T+G9KEWko6!eX>9EDB4=rC5gLNUya1!0Mu~g5Jwk_UJQQNneGW`gL~9xjJ?q z`?*?N^|+!?N|vE^R#8|_ZbVIqu`BFj;~C?4J{}sjii>?-&Fo_n`R6LUQ^Kcl2Kn)CS8C(w zl+Q2%MW|zg>r>h%GGC7`7@=){2LB(;(~P^X8ydzqZX70{7?UvtQ}Iv7ziAt2ox=Z@ zR(lEmpG=M8{}=QBwG~d2M!#{_GsuCz7KT~mY?NRw=3zb-U?CP^F_vKPW6zwgyKb7i zubvq1eX01kc0g*cdI-Ij?e9@XSWeIKAL5+I3-+JwuTBjQEB#h+t$coGL0CnvouU7s zK-)jPajEA&W_T#2m!Wx~=fB+ZM_j96y}0&_b|i8$(#rL%WF?{v>XFZTmlT95;hl(m zKe?mIiFv+*Gse%X^!+10|AqY9{cX!Xy(#Q=UJdGzKr2${M%NPlrh4{&yZ*)@_1`1< zzsdF${NL5;364+GJIP|-Mn*b&5$m%1$eJ0(vtU2{01l#|SpFR6`ikVwlsZ8x|6DyF z?|+P7|0aZR$Z?0!kL)Ekg`;G}>oCGx_qw~fg+5-I7~n> zzFq#xUxz){eexHgt^d3Ee}7Z{_P?fWNIC;Q_Fb&KDU@G+6M1u(B5o?CVFqSlHcBuT zaqRy*vaHm1w=#tJ^w=k~kX#ghzcYlzWNfQkLdN+|OUY$ej*9zkR*$$TwC^klM?8lG z;#OcK(ueHdo+=5e=)I@y-+rPbtfrSDH(`$QcTUJSrVnM}st=ch_2fp>ZY>F$$*rhy zUW0TRGbN!?xC+TdbCkbxLX+bXs1vR?m%m}@oY06SB#=ZiV*cOhyxlQ9)FXjbq|i0* zX6-NL`J2n%yTZJF^ZhgC`)A3VaLeACLo3?Qjt-=dMkm&GE8h{HUAOf5*Ukxh$$@K2 z!alM>8}Ei;#-LwZ67~xpz(E|sVYJu#Kk^9raTMv@{zLpRdhfhBp$Et5Cy=9Ok#XOB zs9t_^I3@lx&Y+;%GhBLes1aXCZ(Q%$USxVc?|#lwDma- z=-c1NU)Bd3=i|+g&W0xS=e}W~i#_h1#zs%k{&h_I*J(BgacocyEy|SE)$;#b`5ztf zb*hAYn#KQ}%1>QU5@tznHfl>tLJ7IT_gHPsq`CCOW_cdX{cQ4ay86&O@%eo`68z^c z^lb?b{FRM=eOOpPUx-Cmj3ro#Wmt|CScy2VXcbwfOsRJd4X*#f{wLG?mv9^-v)VDG zD8qVe#AcMY3=3Pyu7mFXu={s?z3AKO{&%|nO839f{Y$IWy|$sf(ETgNQYYL$I??~L zo5R2l;+%&$_7At7gR(5H|6M8VxQ192xf?aeUO6n(lY@Vl69#|l8=$wM-E+>*e{}!( zy$8f~BPG7aGwMybCgHuvy0+X(*I6UZHBJ+^6Z^0q2XGLFa2WaZ^VqNkAr!%=#Tc?PvOMn8@dIE89)?dl1R;TqpU`n%}mW39!Ai6XZvsVH2ZJ} zMZ)>@bJ^3(1pfDA_IIN>2FmO*tkrfm#xeQ%f9o#_hjj$d9s&GU7g zjjl7+sS**V@8Cy}O@2|c}`}=;ry)q`hFZW*d z9nIRbD;-mXo!E^U)FXjboZEl=>+z4q`}V2dWRR^=&q2#(?eFu>^+Eo_{m19ZkJ9Nz zZWy13OvkuI>d*AJra{kq^%DC2c#LqiO5WKj50MAN9mF9V#^7tl50L#hih)biJ!1Gi z?W0!~`7k^m#5SE1F`nJ;f7Sf{SGDo~_@c1>rBUIO_`%!17f#3DuO1T4kONOn3yAcCA;f2RW+9cRVB%iLa5r$B^StD{cZ=j5r6P!L=tAdv6PsZNhPnr@CeQ zwpIL3{``gg?^66YetKO0|G592?EER1ifQ;EZo)0-(y!jVg8#jYAIUH58|N7o^CQWe z{`?m00Ii3#4;;}xKz0bHR`Mg!>Adz+qr;D+b35+DkCC3`82;ov^xiT2Z@&0_^q(Nd zH_ZxX^qcpQdH=ih8~!)n`9a6!{qNoGix1woXmof~T)FWRN#W)b{8KV9Z*=(eyo~t@XIMP!gf@am4ugyO2RKT-W;}#`(fBxGCRD8 zpD&vd2Aiyv5Qc>fuNH*I{m+y5DF#0-2tOnHjd6XJjOF~=D2-()kMS25h26Ug!t?ZM zW#1lk;+g})!fRu!Ay_;i)S`ZOu{MGUZ1d&DuxWeWIwmyfGixrm)cYF~T2GJD#yUT9Q}VYLzCZHu%-0};pa1M2rrb(44bFT4BMvM5Pos+;!yr0We5AS zecZosK4ScjtDY53l2uvJid~ z__JfG5045Tlb@ni|L9-HzoN!@4bo}Mj0#@}e}&{C9MK8)yBKT1B1=x6tgVE6H)@K2FtPjl!)2EX@AekA^9cou6>j_2_L>h#^` z+XqtG2adD3&PgCiZ$7Gjzn*`Ow)y;fbS&oIBTer_*J|?*=DCi!u49S+rDqlxA3)}2 z8~;~4JiO?CUc$?G1+|O$-|Xoh|yGO@41d`OEAt zzpH~@5%kyo#y)}cKD>_K;4So3yzE3(%5 z|AI^}^8L&j627AMu9i1f4w0zu6gl5k7U$QmyjWaZe}5Et32Mg-373+Wp~iU)(uwuz z%Z0B*@-*9nCdVakt+?+aJ4~N;m=vz3{{TP4O<2p8m$|of?x&m|U(w=uzBDx4;S2Ynjd}FI$5{-&|aOOSL0x_(O51uvY)eem}qd)+_Yh1D+pVrN4$8 zJ&O$b(B}QOFH?_>^|VFm&86ziJM9xh#xW#4n~kwj56OjMqJ-)n7F#h`seX?`x6xCpGON?(S~+(6zcmPVf=x<{mxm& z9~2pXFwS)pyAHBXICIqagX6{@kkwnS2p9XGifh&HZ!ZX==(QV-!zjBVTtaVjOam^Z zUxsGqCHw6maLQf+xLjQOv`OJg@@m9!57(04M;r&y;~sl)z3>l^U8P=Ls-C{vb)fE` z`#bFZ`i(z8(?a79kVG?n=$xBy3x0&#aVLI^d(d@6{%7CvN#`ec5D%mFp!a{cI6O*kJmvl4ar%?U@2BaTPaO6Bkzap) zvGPX!XPy3!pE>4PtVMa`)1l#ca>MiT?x(}T3-lN95?)5#diDRxb7T4%YuLq%zP>~1 z{|DJgyyBQw@mf3E5dI_ngpYB4 z{jE>w=hxpVzgph?6Z?Be{>SOTKgFN% z6+%uKkBc!1m!NJX`-BEGqDdJQ*UinZqf6%3_dm-2Kdu}jJI*M#$n)#_cOBvXuVjB$ z{bT>X{qXQ`ssFhQm*YyLoge$pucr4FC<7gHE&cn*31=tp-^cLZ$?L^ctBd@A{2^jH z!A;~Xs5znTlG2WlDGWanz8%Sh`mGVyE=VBmof~a;et)C>f^er}YM1GEExjoGnBG{% z|94DW|0mnTCJV=ev{ruk>H?bNQzQ;Zf&3jwkU`{0z@xExOq7ZuE@sKZVjQ zmj4Unf8=KI|K;-w{eO=8mi=EUeQ7+87f`J}_aa$yOnVF4{1W|Tyn=6yk7?yQBF>M0 z)o-sMtL!c}#-fhigcfulKVDwnNc4BRg!}M1Ht-i>|9e~$_$}ez;&=Ex(k1dJ{y^_l z-|iVRH2jhNC*->2?N)i5>?;@={wywz5BiuK&_44i`4{{ZU*Ic**L~Nx7^83rF2&%T z{D(!_|K#BYb>v3;ZU0BsI$@VN_HtZ_tFcagig{`+x;(>feSjdHw(G z|M2{M4=LY6zW(LB{zuh6{=)x{>;L@KcP{qfu6^|MhU>(7G=(VNVRdF9Z&S`8|I3)ap{vhI-a>*0wm}Jvj_rIS1znPy*=J)@j zC;0x!6hE2I{sZScj7RY}p2Sb_Gdzo~g#LZU_Bf}P?9*qQDdnS+IsSc1J^x>Po8Q~X z4#&r}{qy7h$+-XDTIrPIdAxvhiDL>!hZpI+{P~_0{7L%D$Su_WzMTKR)PIpN@5cTA z^8G;c+SQ}OD~@{=HFMq9RQEMY|GTy8UlW&{=Dx=1e+CVBZ041{)rhPo3L@7IdHseNnFV{?T(l{@dsMAMyUleE*NU*CM^v&F+7_d!6t7 z|39SNdvKQ7o!I%KTN^x5H;lmskLZR8OkhT2U;?wEBX!disG_&%TBtx3WMLO-p#r{0?h#5#A3khsc1X-9s1PLUNf%bmR zi(z`~BvY0B<6FPqInVQ+=Q+=NuD^3z>iNmqRem8Inw^7Z%YV(`nE}B+Hh{3;1-c_IZUo{ZK#C zeh-sfE5bC)Kru=%8*?xZ^RWPnumsDn0;{kF>ri8EP%Y}%2@2V*!F{jS4?a+4=o4k^gZH!#u)#AnVGCMI*=fl3sq72v|7G;F zasEAIcNhBvQVr_vo$7)Y?C4M2^P%`J7Jrln>-T+w^cde@llcM9bHiXsPN?wvxc+ZH zc@T$j6vuHAHAtWtZAc=8p)1n5JU*~V8sU!X|IU)<(T_{0e3Tuokk@bnw@`gg{vXH= zx9Rnp<$v6z-^c6r8~a~Ec~>(<{e!yFU)Z17)8p8F<5v9yl>1GzvAwbnJmfBy9v+iV zk-VUON35fJkfQe@tr!U%qqkF+7kF{s}6Mkpl5BkrMFw^zI{k$W;4=}kzW0U7)I z%3svK+xf5Nzh3;wrekk}BEOl287M{xW@8TKA^Al7wN<*wU)eu+$A4x1qk6y7`_C7~ ztNDTp=y8tZB610qp>LM>P8VM?j#s=I&lu&Kt#G}v!Tf`*FVkRIOoPkJ{o)trKCW>+ zp8Xf+cJ7+@N!Wec`T_1Zw=*+;VB;HMoqL9Eu+Na2um#&thH~tQ^0+ii_D$Rx_LGBa zz7h_S1DpJRm$SoRdZzv_+a8Y6JM{147}s%n9DC|kN1deCpif?XZ9g{#RhuUb%2#o0 zKaTOoxy5l`>?U@aOte0bO?U$7iW7!VLj_bxXr>H*;#We}Op>$r|t34vzepIz9cOEErgnfzo8X9jZ zcf4z2;AKB$*NW|7_m=I(HnpX~Cn28gdc%FU5X<=6Lz%S0d=$oQEpQXLx z+R%+R!b9>gp5i&eKjc46{ww-83>fRr<&Jx!4U2M92>IlI_GaaUtT2K;3OhG!4P(ga zGv=Q0FQnI#b?36ec=|-dJwO^;l>hEYsIzKrZucL29O|+@4)sW&0gY(F74I_H?}{)D zGw_$z|4sckbPaqQy3vCadXYvSnm_nBw0!h&IJNZS(7O2J&^Ghq&_3y7m%P4W!&XxJmb4w^qqa1 zhO54hHe?Ii)+uxKTG2*tzi9p5Bm3JRsZZU7Zv4vme$wh5VOC&2R{5_El812=$5ErL zQHwftU;Q06$`|I4k&S4&8p28Ue0hFx4R->~NH1bHE>oA?&kjlM6i(wT&LeFN=HRA} zLqB~WxjkGWuizSP;1+Jwo7BX`+ncenMHgWEn1Gq?xJvO+Ogg4r0l{k1TMY~7d@=8^NU0E-a!R%u_U zA4uZZDiOr!6S z`fsNCzf}ECwic-WN2}Y%ssG8&;nu#LSQlFLr8D+7`wMoQu$P?YB+%Z%Zg7X)fUUoo zyA5f2+yf}}gdLy$lV@APkMC@;pXAnXbi%gqqet7q4_9tCez!3kzP=&+U|>V|{?<+5 z(E3f`AU&&<{;lS24VgI=QJ$yQ0efZX$KL|--r?BgzHDULm&7u6r+OQMfuG$>F z)3rIAcI_;_yKzJKY{#}xF>7PkhwpjT=e66zdDr^!{k2=e4}71416#tO?OW}AnP>f* z@g>);;2Lh=Xl)3$$Vz8u{nY<)0=Kz;)-pU)+o!V1xoann7}Gj1THkDRs5RbKvvGK6 z8aE;|-W#cp_HrDs)jGISj|xM}8MY$!qBeQzu5j81#<0bV4fp8}@fc6>dVc1Moba5x za&%q@=fr!D{-d_)FnaYQ<3BU=LO#8IEgQ-$@0LccYn#x9BvLqyvpA2M z>H5d|k9Ar4$9O$IUH|g+{QpV%xB9WIWZMLNcXXf=$#d)j`q6O>e2+eLitOc1+bf_C zt+j84e$TmtE4YS^%fh(+X1GC*X9jk^cr)Chr;GI2{5~DSa8JrTvENzw;LVWfhc489 zJM?C_?cTenI_AB*y!TH1c>Aiw{qq`kcyAWlUVz!-Y{dSk#Sq>o2asXZvsusEMB zmA!1Z?~A%|-gC6~n&7?Ah$c+*o5?7`G|WITN-!HqZMd$h>KOg`9x}zF~Tx_K7*dn1}hORNgEgtEvk0A)NDf!<;(y`z8G9?kS6qFlV5_ z8Gwywx@kVkT=mgM>LWIQO6hG`{QU>cy~PT8X8)g{9-gWmo+Le%%3ow7nnv4mV5vU9 zPS#zZvum#&thU9Af`}NugWY2bO06f43hYNmfjo+X^vwRc`pJjsN73hf)ADmK;{LkF`JF@!5{ToD z&15|Lw~frq3)ioRc7fQBPV!6PG~)b&xG&_uLG%BMVg7&Id*Hll{kViHxP}|Jh1HjX) zM?~D~HqoyB?^XYkP26Yw_Bp~olKwFa`51vwNUk-$j#uOSsj=#1qzly3#vGfks{geK zPLZwdYujpk9&vx3&XvaBT^l2uLX5{mRF~?%Vlq9hiHq~!i|EtP)XV;k1|+iHFu_y$ zTQB{s)_z{-x!3%^8-<=(YyJ3p=GcO@M&ih>#|3mB_$2^1l&&9u3 z7$umEIhcp}Sb#-HZj}GfjUK$-|MjYGEH7r_-y;5K-KYKIIqeJC|Impg!l)F-W#kG} zYfCNiy;sqz=yjvL*G7GL?sdqt%MNJ6#XkJZ zZFI*z`W^Z6h5Y$I{?s<_!(G?z;~^g7DV`&om)=mLOsK`{@`2ufMl_-M1Nr+S`FpOs zyjY&w>iev=$ZYAT#IvgGUGlYm0c5F?ps(z4)ff6R8PqcBgiqR zKfyl1_SaCW@888fafp2a<;D=w^TvhUtBn8Ow~m($tI++i92idy9nA?7$;l|fG|WIT zN-!I9Fc0&w0E@5$Lv6ABJ}wMwlRmiP89*z@Rak>{Xw@FyKyJboY(t#e+5RNUA0*$O zkT$QY4{jHRJ=}c_^eW{*i?pggP&3;2;aKB`Q;Z)Xv%Y_l@x!Uc56Mw}^Xap9jLE+v zQtyOv?h5R3KGAM#DU)Z6ucI41NTC;L^r3mV@k`;H5?OV5dG)H_jm;_a|s zcn5J9M^Rn%wsnthhvW1(CSG^s?QoJ_gQf@G?T#_dN8a@r__ZE!&H&klj^1}dl1!n! z-5Kky_24x3S){ib_eTo7sPRs<3q518_^%b-R$-%&-h}gh(~nEIf@`>eTeywnb@BI3 z-QJ}Kse9s&^fB>2B>rQ?AE)@WqRl<+=%9C^^5)Au3|_4tx+~1Mhe4J2-KW>PUWb$N z#6#}5cR?fK-Uac$ihE~8+fzC}D?D|*gZ`Wh7sOxr5^YXBldL10=l?Q)EKTp5>c5{Z z4ItNj!w~0Qr1kal=|eT%_Bw8nlZOXp*1e89TMwpf%m02OjvDV+d)c#5U!Yta zt&g0iZ=9u%j3u783@fk(@$A5LyBtSQ`mJ}+z$)=(;hZDbkBqr1ZYTVxNAhi`{H+=GSAJtqhB3HC>~F~C;e z>LC3vj-s3mpyQlp+{_BcxqB{qw~PAk^cwWFNCOxA_h-B-S$RD#tm2IDo#K9WM6@Bej7`|m^Z z0~ebgxX@fJb^q)Ai6?1)Xa}^aV@C*Q6vm(saSng`fj7c()mdv&gI!hH>0>e_qkbEtb}{vQ(b2E5w; zzf2o##|!P7Ga<|oM%?pk_a-(c`eXV0zqtPKfW8v`s{Q|mj5*I2)&eX-To<#1T!s}G zy!Dl^ij4NJHPLN6YF%_ovm3}b|7{bw1>ZLhZX5Y({#zOS)%>?|`X2o9{I?2z`*9GJ zFSN7Cst4Bp>8l>4AIC}5DGOp5kw7dXV%gAyFV8QiabE(>NH59{Npja|{p*R!1o~;5 z#d-AO65`lHt$bdGdL-oi2IW9z|K(Y+{w)Ys+;wPeBeVbVLiu0bZ=E9l3$uNa{EyDj`u64=J`~QNxrC3&r-*AXo|EAp%g^iH3b|x@ z#am$*nU4_|(iRv+mW`G^#!4Skjs5iNL#)uA<{#T1h4DUP1>?!g`3L3>Or{rM8fKtU z-?PoLCen*hg6aa}$CyozebRU~{T%u{G_BM(-L8*{MAn;OKEK$9UqCKG2Ym^-4DIx8 zYw&xpf_oLx^gg`WC$MI@{_lFvSgrpX!z3HH8_A}wx565~S%(eSge};HGL$1}jd_DYuAn1NJ*jTIM(Jd0o}FRsf07-*JpX0FT!B?sgLT+|E!c)K z4BfW&-`s$5dgU?w7Gpy@*ummnkmX0s-P&mUzdbwbaXqvDkGSoc=X>_&hS&3d#5;jZ z|F=p#rLSL4#yRZ`C)DxurkihsO~RR#N6WX3r>r>0i_A9dcZzJ>oq z@t-dLp$TOdv^DW5Gx;sQPjAsC#clds{FU=>7s`J)g}7JDFVDY?XA<5QPGg# zS6=Uf64T@m8`#b{u8+{BaR1=7-wC@Gl!o2L*p_h*owRNw%Gudr6}blMumPK}1?43r zVH;T%b?m{Q@yH4?j(@i<%nAGH2XPoj(Xln!fO5iddhGLePiGUS*Wi`itC#ID7446u z^8XI`|A_p5K%USZ$ejNv&o-gF##(YD{I(fwNFs&PQD3P1MK_|2Ew#~qgEac^E9?LH zwd2+Ke`W3RZm;^AJTJUm&DQ1LDhd7cp+#(TMb-r`lBT%__k?hTyoMXNh02A}BH4HC zx58cWKB~9M{~MLV^!i!iHa$B$rawjI{6A@=VYIkn|I~@$K;Go=y@JW%^Xrqt!JSjX zp_xVD`{#%@BL8w;(cwb_k9EDY~1_$KhCZ% zcwhhL{V>OG=3zb-U=fyJ8CD?azOK>sKS0kU`yZ6rkKl!G{^Y&T_u#$Iyxsl>AK3o@ zt!P90NA^E(U+2vCL*mESL+>H`KUh;)JI?+C z6YM`wYX5;r_8*vP{{gaTsc;MA4{hAoHaP6}M{yiAF|4G%zx>*zJ<>ha_#0B%D!oX1 zMtn}Quv%u8hEr(GD-CU))qX)I(tH?E|^4mIXkGQUX9rp$_j??~lqMb28J7cu? zcj^D*_4!|=%H~P_+o|ela+BYD*}k%cdmGA--e6tf4)cTdv48Aj|2V?_v6uZreUet6 z^eJbWmur70H&3AzZD_w|e$i#~i;%G&R%M6Get45#=8HYTu0T8kES~+UT>qGj}$MmoF+E-2=zH6uc`M%gcmp*;BUZm-L*l|pKh!vg{+t{nfHCTrY z*cHd7v==wgw_qE}P#%fj{)j$2y#o7j5FN|C0}j)B7JByfx582Sar8~sm;X?o-X3+) z22r_G-<*F95@<&CG4>C%(d!TBoA;W>Ku@9Rk^cK${r4UE>f~vDXK^0=`11JLCGK)| zm@DM&yKyhS+;EM41GjJ+HPT3}G*d^`Px9ZC{^k6C*FE>~5RdT`&tcJNSuWUULl?TQ z%LDTBuk3#z%ob(IDY8}AZS;6HOo!)nK2s+MBOfEsdcfLU^40#dW9Wq#k8U=(IEN(l zLcK+%5!bR+KgtdhU7w6N=e&rlypSEHku%WPtFGFoty!h+qSv)+XOqRQmEiSyPyhXF z?zHwq`IwwAk3Js@un0@A3@fk-yW$*sV{2>Z>#zZvum#&N)XX+WMq7C~8P^5xAuF&S z2XPp2-NRAxI8LGl0}1~F8SU$3)1}eUm+g%HY=CVdyXH@#;}BYo*Z0YE)01IS8xqCa0|C_7x(cH zk1=%ntx&z1ZH4|ERqL}u_$R)9gLysVFfxg@PG@$MS-yiv|jX=#2^)Kpb)l*1x zy|ha=(wi{KJzuVW#&8#6Jkp!AzZ%s4?dnK&gB~))-OJXVMjx6_=>OvsTG58MF0P|W z{qMPRymzz##QkI2=pChO?ZTdnXak6S{~qib#_p~Elcx6}wb64B`~F4lorW1GM)g+X z7p|4i>xCU{0JG_H5YNvgEm5y_uQ`k_d@*7iT_3MC);j{KiR?ENwz*PwlAD@*nmyw$aBxg zH^LTrX8q$V>(c3E_~rGF<@_rBPkYD;RDWPU1F~GYuUaf#+r?|H_fZZV=HIwnytor& zgX=X%^ncGt7pO-94f1cJeBFc{#_G#()Nen1W{$vq>+k*V$GJ};dCvIyMSX0t=caZ6 zz1J9j`iXYJv0seO8>2skRec$Pp`vxzC zaGSh~`xq*=em2U->P+%!)SsGvqAq*YKPeC6!gK!NpUN8}#)Vgy( z1j~@DHU5q6<;K|;8du+FTpj7n#@W{zmw#dW`=ashbH=|LjDL3-|3(MB(>Q-CeT8sV zVGR!YejVf7gLT|J!-YNT?XZC!_wVo9ZfqTy{r^WAJJ*J&eDU_nd1wpG|6A(1dSR3M ztLSy^t^erlm*d~c^G0DcAz{2fv;Y5cw+Lq&%Hs2}2Nl?l zgLu9EcB^(CQrd#O$h7mW%KtmXf1mi1t;fWlZ0GJEJGI4HAG{q73+E_~<0LxxwWEgK z^P%{E;JxV0=;Kav_mZiF!sAz2Yo7x$g=+d~@+_+8b$Q}dU>*Z^KN`84xD%trldPGb z4URh0BY_4qq6wG$<_fMMy}~#Ec?)-OAIZJ)Kf2}ho|q@QRN+q&eyQ-uH2OyC zf7~;V+4Ws()W6z_;filCU>_y^x%B$A;0+ljYch3PhXoe)1p=<0vxg z57a%!>2XY=8z<>C=zF0KMI0lKXT@jw-|Clm#$EjZWk$QQ*pAjRB;|zH#dRy~-|}(l`H=`G4-IRL`Y^a~g3T+F7!C z;mi4d=jpHZ-^tYf+oZ|fFU|j>_q*p3;@&@3$hh~x*Z{d7RrI=Z%C(C* z;UM>6G;%j_CzL-8WM+TwXkV0n%Y!)w?in~K?~sGWUTeq%nlUtu?SM=oh11w|%lv}J z+VS-B=*J}t^}i9WkOR%qBN@;3y+Pi>ZQR9uJj7!>#dCyzreB9)i2KLw9AkfQVdZlV z7HV%yQPzy}zWPfe_+|PZ{q~<*V;?}{`vaH$NlwQjeY|JJc3khdF8^Pa25y^!&K=vN zY4jqVRa5iG{%NQ~JrdLPZPAD(ob*nk{BArZVls*_4Kt9uXMR7r^@V$OnisF%+^cV# zMqj(W(qi+^mz#f1wz{wFnfd4Fpmz?KKfkpg6bq*WvoQzNE&AVGZ-#mF`Xl<^2h1~| zFF@QozYz^cc*Y`rt!xfU$YtobXRZOc3hnf6Hj5s%iW%auhF_X(qL0lW#lQE2ed{L} ze@7jD<@`hMyv}bnU=y|=mhn4=zpSseaVKYqKe{i7KjNB%Uc@v0`p`U8{AY^4_syJ- zFk1Y_ivMu&M`i7s4iJ`4%hV5SJUiJ4svE=~d-&Hs@^1I6UzlusVGO(2P4T}h{&&Qm ze}#KmAE*P!gNSoK50gg`_xa3>ORtxnxlbZZ?_2J_k9|;jjXC_a=JnT2kpGcD1L7W^ zO{npk1e(!?pN?cR$8qm{Lccb-m;D_XJK-UHT7CQ8&GP7W`JZgQWBmJp@o%#Ag}Hb1 z_M7Zve%tBZxPQfG=e`+|!cO5ds!y;V@;^(@>_6%~&eQwR#0K6t#XBO-A^7nmdpPFCHuZQnle%m;gIj8O4 z3cus${ej)td^8Yz$0J-#Gc%A>XP2*hka`!(*zH1{e3S&@+@u<UTkn>wx?aSeaxnK*|~{fO?Z+6Tz&zc)$UJ5n87 zp#J|bFHH2D$tc1!RDWds8D`MycdGwTOfNx`y08&(@3I7D^P7Wtn2!Zmge6#p_^oyz zmX|BI~UMg5O-e@^HZ&Lv#K4cx+Q4Ba*sMn)UneR8nK{0Qv>Mqq@U+hU;lZ| z&$6E8jNo@-v^`SC>06X4V@CeRoQAB2IgL~QeNI!>zt5=|`R6&cNB%sgF6(hl{kZ>_ z)0TBNr+xLm$w{Amo>PwItou1Fi-&SfeKeHQy7AxUj1t}$6kLTu4;<$U;i<~9&YGrJkL%EDz zzutd~74%ie%&T6i{9Gt*E0yp2zR0OP^hHkHu`hD!$wbQ+ISsvEvDkH>!V6weW^OT!q3e2hTyL-S9B)xF$v$doX9 zH@|G-MO$;TvcKh`@`r4_sr(_^x#Jx1&h6%Zx4+04C7dycHrPV4`ofExDvYPst6$>W z+sX8_@$n|^IQO=JjC**NeJFh{_J1I*|0{A|6(JrZcBRX!b1KB0Ms@@1d$Wv9HZylUGkZ}%!+lrf## zm0#YYOZm`E_AFN>kiFb#vhNvlfik1DKpyg}C5UzBGI9mt8o+LOq(}Z)#k~e`?O)$= z?VQcpIrMe>HeeIBU>jopp^S`cfy&7}sKAhVCo^C77wz}`{4(u6bE$WaVH+sQwPszK z$MJZ7jea{hu<47O8glTV^g%YG4N0VM8fVdZAS;|FU#&0er(ePqbo;M+q=(dc>Eng+ z`H8%6Pyc3$`fss%0vY@FA@$!ebu1b82|ehy*Kh;3a2t1V9}kh4|Cgt4tuF5&Q~Lb9 zWM=%mRR8~p_JKO}6xq7k`1^L_?`#Dfi1YsjN92Xa!Wnp&8=jJ{+t=#s=iK3+`$oT* ze~?Qbvc_>3xtl$9m-cX_cg^P>fl(NPYISNA3hDJ*#ciX0GJPWAp23aJ%rSUmt%3GR z%>nU89Wv|xy2QUq{2Rodoa{G6n1&fBMhRwP4w9?oKi@Rd&i0Kn`}%Qb_W4&AH{TS0 z;hYj)E14PpsMS6?@^btm?uj%{IP1)QoU#LL^dy?-(FPFr7>zc76u;9ri;2?Pkny|o^nP5zAiK#Ga-bqBTqAGb z7VhFcGW$PoR{m{gKbUQv?L+o}Ed}ABYmf01&k_EGvKzyYj}aJ!F(|}%OvGdqVH$?| zwa@SC)7uAV2KV3?dytSNn2kATy<;s1Szi8Dm`^UitMz|N=yC7K%=*9M`C%D%-vnuM zmNHnlDe;HYhJ*dE-c@+D}0eenV-my0z{Va*5t^RoJ*fzlOe+!7ZB7SiXylG@H;@P()9@*l6)|J;!{zi>`w#D-SD^2@cf0HzNBe$=XTHwhzaIy27}e(LRhj>Hl%CoD|508z zPCtnzby1u{-{77E$`)$>Emi)lWQ)*dsBzD$_4f&SX8qqH<@W>a19fi-ao?WwW&Qt) z+6V4wLlP;R##s!_&JE|ufoW{~FaEZKvYq)`1h<)GN>y10q zX=^;CKSz^sfX3zOe|`zqYP>_Ocd0{MC-9ff-#unt{Y~i|!;p_J=l>Dh$-V3w$c+DM zL!`RY%}6iQ|3~v3_5TC){|oiMHc-Ysa!&n!M*Z);8vjMD|DDiZBiF+};`FtMfIA=_Qzr%=sM3gVZs3Z>K!y-aay}@tx!PJXFco^U2Dq z_M{~jp^@Hn^JSZ&fvkHVKVD|9e{5e7*K4L5e?%SXkvL!uAR5tR-rh3ztiUR)!8&Zf zCTu~n*Zil|=0B}B|A|bghkMC1cOTiTt()0j?V|KqtAE&K4q%lzIw#BlL|L!(9mA#H zvDPjMuMFuW=0lFMp8u%x*Vl~;XO(?>_*cYhslu?IJcz?+eWsm49>+=4AnxrD_jpa; zFedKXcW?E-@^40_{wEJI$9pSV|_m! zazDmXJV(v&ckEAN5BzcOg!<9%ghYY8?~U&_q6y6--!Z@Jop6f%wDrO};_^;tKl6_C z{SNz|wfN_p;m*JNs&7$ien6M`0Z7yPPWX*A`r-eP$1n`}=(sG5Dr?Z`J@L0s?7vSR zgFeqm&k{zdu-sS3Z#*VqGKw&;;+-&!9BeBLGst3;U?_|Wv&lJ_hxw?S`F2=9F2WKl zL$z`Bs!8T0(Cdw}*KKAeuF(hNZmKdjpv%~KuQ7D@m3{OV{omFL*2uc2-FNMvucPle zuAP)C4X6`(8vN(&{(Ij(O~$<#YL3YNs4I~F^ZX~{{5ND~{pn2Qyf8Ll3$~#Q<=BG? zBqz!LQ-wcE_()9=KGHX(|EtpfNAkaWPPwO*Y%7&E$&Lx~zh~_i#z7oLJVWXzS#?Cd zrXQ!DL=EcX^?G?X@kG7(;$`2UDY{*2MjK-LH%X>&8fTHQ7fN5Bj+2hL`*8`o+v5J; zxf%Q4HSQa@g`vCV!IO7!AA`@$fhS}7e@s5bbA*2>&7gIqv3)WhBQOfD*8hy5$3A~I z3hCp~H=t}#CiEhXe>ZojpK)rq?}N6nzE7U-gHGQnNoLMhTkrd>_I+3SzIe61a_6$I zgo(nPj3SJZXQq)eP>h=C>=(0?dl!5^B(D2@XrwnG?%6X?_I4<7?_kPV1ol{%9d+p@ zJt8;EqtC}szH~}1!V)Ztew*Gl|IPZpSOyN1a)MasB+{Rx0#1`}Ogs}$eumPK}1=~=Dq&l*zLH$={{Cl|i zeXRN&nf-tA)bE$oe>jC!{POvLaZkW<;p{;L_M>CC{%;oB1HEUouyL4v6n$IOtsB*` zdFomm=QnUXH=HB~muZg__!cvzw^FtZehDs+GbkMJm*U(OX zW&gXueG9*0|I;6@MP{91sqa6@_n+$ff9U&vpl#zfcX1yN@fc6>9O0Jl|Dp7WxVKYg z{n=LO6KUxu+JY^MOI@J{j&*wmNSN|E0BKkq~CGUFWJc5 zME<~-+V@u*TbnyF96a{*@cG8Chwm*OZGXjole2&6pXK~y_vOU-+lYj2wsAruOKaQ=H?JXzV29e#S%eu(W~3CH(JJ6Xk{dH8$UmhXmB z``$Hn@LQoRYofJ3cF@~BkaET zjZl8PAQbuiwr4G-(qIYy*_eYN zV@~tPxc+ZG8TbEQKrX@(EJHl|a|O8yYp@O-$Mm0$WQPs(p1t}{z4{3BEy(O&won^x zskYn!?f)qSVH^K4lw%L7ms>}$v>;T_>pwI$yj_2rei%(h?EQAY9&lu0VZmQ+|4%iS z+CA-L#|7)(>6!NbFl8`z&pGyiGwcIMqYo3^a}Xy{g9MtAFwsFqHV3Y|L7QJ{lRGK58T%^lYIg`d7jU|_mQ#{eW+ZiAMH8i zXS2d-@+_)%N+3Zxxju*~EOhyqp)b;I$^r`7_&RyL5eg?f5 zne*?au)!g&0ZRAhhDv#?#I@O|KBGTC&O_B@b&1Vw_-_|~vX8r2 zd|RG~KU&X;zqq%HcSpPU>kkZU(FWJAj_vHh68&u9uS0rNZrDJUC$hs9a`zVY`TV@F zjb4T!d-V*>)Bk>?4(Hy33hYPb4Bi3jqUdo?xbC&)WYEix=Z3yn^7&F_v_3*?OB{{< z6}jPfl%Y7BBx{gBGwP(jm)Ol&+YRqRUaU7g%* zWcy~eqvg>qRS=S7*WzfGDq#07(Do?^y-x~4JpXl*=d>Y-6spZdj&;CkdR)6!H^n-8 z`gz1N0UF1uqeiN$)LZ@hF5wEU;mh{>8{D^W8=3t#*$eN|@8cmJqo(Q?=lk{Qe_v3i zURSrGk=}GhJ3>3?sr#NI{3~rf3`0Ifz*_9kb=Cim9%($b>3R?O|Bw0q$!2wV%WCWY z^!Ho0>ub^5*BgHzJD*4sGtILT&KMM8JUW(Ye=Ib2n%;9l9sZGV0D2MnYPCQ3^^&P} z`~L8ofuVn^hM)7K!{?xPLAfsD3_b^Ka;^%KZV=uoF`A-AEO-o046V0l(3<1WW5 zyFlE7zL$NVW~25g>QIkgIsYb66889A1@_}04&x|}Be~Z28oGzG&m%QX*^sAPC}961 z|I+z4?t8sIAH7q#a8ejGNT5>RqM59kuK$5HdJ-wbvjO560SWDnxNrBX{dtGRX#bA( zjpg-di$Cq&^Vs#YM7e2h0sRuL;2H+keI?{dO9N}Hz2*L^?0>i1a~pSY9}n>uPw^bd z&DvkveP^=gqCBk}@4YFHU(o(K!~WC$U$FoAUphZh_E_5&2EEr~^FYb+F{NP`nU4_| zg)u0^cud4(3|XsCMDExw-Wx-hMvrq`ipi3AUA>WBWbcLjSz$K!pgjcVkn=DfyRP{@ zkFA+MRvONV#|(Z8un5&xOWFTQ!xDP^Gi$RSm4;>X6=)i(T{ZH((BPf~R`F{+@m^R% zu0uz|dtn2)3GFS``hRF$I<|0cLz>=)6nas!_`OiO`aN?L-V60)V!N={dPbM^?bmvsXrs5I!#$lF-_N9- z$^p-5QKz8#!TaI3aMSvmRgdDjeD8(w+s0z~H`1GsKm%HDmV_GqnfV3AYZCMh>*Cv= zl!Rt_kG1pNE!L9LljvLO-4=RB_oPs>Q2ddJ|4QMa0k6;hIxbE5-D#Y~dGzBFt{|x{ z>_V(#dXSRmGwpwRADUl$sr|3MZ0pj$MF+i8+q3mfX}Bhw8@Pqr=oqj@-rD`U^mrz3 z_pI!2pZ*Yi+u8omyH)#fqO_w8mg-c^#TUS{*;9(Zc)D>)t$F&Raeh8ZYE-2bP9 ze6|12Z2BC`!+-1kKXL!T`F_6ui?9UC5ZC^#Apd{w|1$~aC5$;Ia7>0a|z~H9bFp7+O;*B8-F&+~!8AX_e87Rh3 zTW%;JXQM`+|MmQfTIGkl*?>lR6XN{+IqsW>`B;EOSb}9pu9yGy6T0;wdyt~{7AX6t z%l~M;F8||{`&w7a|LUrCvICv!pdGzAY~aS;jpalec-)(Qg>cjIUxzw)4SgMY)Vb1t(*`dnyUn}=%Px9OKlszS`*LcKMa8F*>9!`+4EQou8ZE?>w zl%X7ZP=P_;dOsQ0;T-}7yr`9?_7`_Sxq3r->KmDo1&<=Ez{{r|5C?*?w+Htym+9%AT< zJ)p?g|9?sjp0qbY{^;AL*A|)i1fJpF(zC+t$~1$qM-5!X5xk9wW{67FwCd5QdOU&kapInMX@eYz+4{`chn zXR%%HI=v6gANu|aeSfr~ZKk*hw}b5TyUM-#g2LHlY&@R-ahhK3Op+@8XX*8a*njrf zBaq&Yrf2Lw)`>LiWdFe>eyzvk{dw}$?A*|?I44};cMX~SLpJIcm=wCdOhc61}oZsE_J2K7PN8T3ZUEIe*?A$gs zJSLxFm9$$kLH?HS;vQJ9?U#@AAJBy7?hF5~yo6yGT428^as)M4S?YCdjJ6hI-wnpV$;t!T7!}4RCdutt$blJo;WjzwOvk(AeLTctRE~50-qe?Eh>U%YU-eb* zug&qCUjNAZ-_yst>wlQ6PCw(j-t_(h-k<+}cKw0>qEWa_WcohapWkNIKaQ6+=_4=- zW02f#{KbFRUEn_;Q@(NUaKD>n{{b{>54CLe?cLW(wk`D^%=PX4znx^w0r}&IGJu`o~)SO=W3N$H8V%~2ctzqY`_-mUSobh;kb~Lr@HoOe~~@%R*L?5e0MAR(?abpyx#xu zk@god=da%}hyR+kg?evCmHH3cgk6Sm44t*F13ADpP(cnBTO(k8!2akbEgd8eBQxL6 zw<(`psEua~m)~)mMBi$0K<`!WhF9x>YFta8(i!5L$Y!)5iE3lwRaMR+r`L}*{*BZ0 zvuJ96J2avp&-gda^XtbYT)~&e->z}rz%8^ZFE@VEd$;+<*R+ASH*?n*|F0eWrTGQZ z%`Y&v-)K%j(}Z`}NB-jet-tj9?cDZQ30>FO|Jb;DkUGHrhcq(#Ukjr}n5WQ+w&CU$ z;MMthned%+Gqh-2@F~KM_>7ciUh-8{3mXi6X#oXS8^B)bPELzZ(v$pAz=t2hT=^?>`zD4(%+mc5SM;0Bqm~+0Y*SR`}Mw z-wx%6emm?OC)!|o&pqTlzPasOt>&j&k4FuZHjBhwx3!v%iBb_xFK!*@4qp;UCkJ>%SWQguH@( zj+?jz%MQyX3+G?cyXXTa-wwaSJ-Eoe3={q^^xK1N$Td@9%l;jnKv$})zsz`o^#gxS z|K2F~tgwbLWQQ-fzv}nNM_&yE*BtJ>`Tg-5q3mDc-=UEEKauZ{U;Robn~E{`5BOcw@_(0XBL91`T{-vn$j*(w zo&ER87Pbgmrj@l8Ov&Cy_TCHOIQgH@fUdE>ll>3Lo~5I*-x(Il{;qc@A%EAm{2$3# ziozY_M)Dt#zeA39Pbs;K{M+RJM9y{Xf5nH~jy)**JLFdWOUd8#ZvQjcQ}CPFm+1YY zen0yi?!ML1ihS0~R(!`hl&#gj+id)GV^;Xq%?a7%Yrhh9?rpR;!ybkCuMZ^&a=;RpKNhc9P^BkJ8B zs(*i^{5`t%4g05#%Kou?euCqhM`fSrdL#VwLu+w%PR%|!f!%Q`J0$;~wX2^`jLfcG zI4V1yPt-C&JW?~WFP6;A_WEUYy6y=y#V)*SXYnuauW-Tjw~#!cZQ#G`gsP+LPh8V9 zExVcLKjqh*r;NHPJ>0Vg%?o?aTu`^(`9^lCKwX=q-X5^W6`J+yTg(;hu>P~ty=_nI zLG!@6&{A#3GtSW?YiHUw@}fB=t|iEZb9v#X{44YF!g0?yG0VJyae3iq+$YJZ(e}IX z4nOky_sGTg@y)#Olj*s}$+N?;t9jw?^V^G)-g(+@#;M@yHu`7&yMK#+hyDKZZ2$io z_`dXikh@7<`~%m%gCF3hsEfGv=hE4)za@R6Klg8DUu>S1-9MhZMUKE4ESZ*lu_Efz zvoDUCp51>X%HPlK-!MM=;-$Zy-9L>!k$-+k_Qf^tXZMeJKl|d0C`+?1p8mb;{y8P| zS=s%?zn6WnhIc0oaANkw z5fiif%m0?&|1H<~r}!t_^VIWi6=z@ERGj_S#o<5rJ}ag1J-9gjxBd2avg`O2`Ck1e zz3*}N7ynlFALnuxzbpL+Z^W$Zi!0vC?)Nd%rN#mu6lMQ2*L#$yxxP_X>2GHLsW1%1 zl~wUK4WjH6`3KUC=7{s2L)m6}Q?K*vj{J@6hN}GV54h{k{bqKK>t|j6Yvd)8S*&ad zeK{(##$+Go_f6rB!%vpyhaWG@H$Nml{B`ahEuE0vM7E#}N&Ghc1efsn)Zfbfr{p^S z^&jA~y!`Opjrn2U>in>k`#W>TXRq-7duRS`_WwfuJ#6E@f&4=-b(IC*WrydMtwfOj ztt0w0-`hRE$h?8*HrCf_en*@69e>K*G2HwQ z#Ph?G3d`hN-=F=J{_PT!G^h@M8nEN^8U5eE@38;D?=~&+N914q=1JB8{PG;(j^Co*_nWE6kAHBB`p9o44IKM; z7rz^M>z}h9Ne*Q!7b-){=lJJ#M)!z>Z{nMXKsP3aIk-j2H+nk06y|y2SF;uzDsFJW zC}@1EB%8i~Hatjjj;*5q6)d_L^rEI#q5p*aA-;e57KNO*T@<=fio;flf%CdchPJ5E z`BhSuwvxW(O8%+4QYzD_uh_p(9l26!bX<%1IAdpH;!3Hfuh+o!ja+Zy`r~)ww-z1K zeXGPmU#SH-8oWZReL=i0>%Yj+Q1tV&vGd(@7&#+6mHr~n;dI~^jebDHUjQ^y)zM2O@WL--GHh!o6*h(0=6DeA1N;3i0dhu~*W=3FJ^yModG3YN;f!BPd) zm}~Y0OKn)N)W!SXKUnIK4WaC*2uBx=wyz1<974P32ocLN?c^fuWB~0VZKI8Fc4QmZ zwIdzaJCL0bw5$3M-rs*dGyei!4w2q~5b4X}eB}(WwF*vA?oIopE8@+|*R?mHyU@jh0}w!7)$VE!8v9So8g+T3*kv@drD z$<@dWuo)PFYHkF&mzsWIE$1u#xrmuzm}dB0)_2aXkoPcu1n##<|3>Cr3Yd3k=f6(4 z4IDWnt;d685%ZUjBSG>{q>l9z+Xm`m;wrc=NJ5Zn;Bx4R<=I|DM`R;&J~1nVX+RBI zM`;Hb!|q!l?;s;-XZ{NwgWo_2yhgZ>A(*h&09_1?mv__7+@NyiZ%~?L%E8hN>O#Jg z_qnsH7zgp)lio#80D36Vg~)fgws@9uU4*A1#NX^Sc|-k{2lUMw}7`%#;*MO7UQ zmFmgOsvdj8^!2I^Y0O+9k8tf%(71fDWFgDob1?I6R%nO0pD@-#_5vzm=o(QD!e}=z zd*OL_5nh7hZ~{)kDfkP#4e!DSK$l;07CwV7;45I*pwTuSk|3mc@gZfIen_<}JfvC| zGsJp2gR^fBh`suNwD}I8AACR@FhV0<4{3co4O*B>Di+)zTLyxN8$59q%$OY!_+F`PauCCJj5!%}8D zEalF_Qh}`W9hNF&HTIhHLsA=iNb155iIMlLUpOQU(71GnvH2ls!rVN6NX(Ik#1ef- zTA(%YkXVs6uCpWCQV&V{zC+?5t`1};>Dq;KV(&)w;NKg4i2WC&KXQQU2a!Xh^Ki;< zWMnEsMwc_iwU8n1r3~>*XNniI5BIUj3>lx!pdcKO?8F12-E}~6b{>%2qywT$J|KBZ z&q#jO0meVfYhW)-IlzC=pOa$38BEVfNgwC-L_Q~F$#3N#W)0*$^IOUH{#Lp$kHYit z5}bnf;S5~0m9YhEfr?)qlpBz@K>2+K+9njx(PzLTi$XYJ* zDeyA90)xLeC__O9DG)0p+q6Qo$Q%c{@GknlkVnTq9|{&$h#m@;82>^sW&@P4hF%I~ zbkxhCg7Ir5RHX#)-wXbOfw>myG8z9uJ>~{z#Lomxn47`uLf0EwjxqiP>-5)qm+Y9^ zpnZuo3h2Px3I07xPUyzm1HCThSD}9w>)SAx!u%=>WA^W_{LlVO%pUOK=YuiKhS%XucnjWv&*92jX?q|F z?t(lhgbr}S3-AtH6wka0JP1F9U%_+088DhWD1;WU!wc{-yb78t*it#f*2@P*4>u+TY zsLJ_YO9ktUl~9$khWuYc{?j*xy394?{~Gdt4f)SGUHF-x33D@;U7RBZEyq}k1nV@q z*I>uo2JK6%k3t9LPUynV3Eh}`px4FvCiL%OeG>*#Sl@(U%p)+G$@(a`F?+y^pAW_` zkNfSJLE?ikeUOZs=vQV0q0bk@dPb0VU~qqs48bsrKri$`KMX(@VVuy7xd+;lg2VwG z&hR;Yw)i{ zmSHasroS0Ve-l|8M|p%2D1`#hLm?DFF6bZ+@<9XHpoJW`_M3dua2xyz4!|MMLLT(P zFpR;g@H%`7U&48a+Af;!z=QBGJOPDJ0!{D&yaI2*=WrfW0^c~Sg3I7ah=Om!ci=8~ z0-l7Y;1Fmb4@#f{YM>sPpc{S$ZN=C1jQg00ZhhT2{8P36D+&9O(u=W@1tLHi+{iw0;;!a;m`*ZHaxZ8vK z8OG>-ooa0^I%JF?=jb5L(K#H)Z;pK$zw=AR$$#)m*5Ur>zlJ&dKlvs0Gk36N^GY#UOOdHty zb$BE8v0U67%=H#4d6FH#AP9AhmFt&q`z3Qe zR|iRx4l`@k{lq&=|7@&_@n1}cTsWVRHC}BoW55rBM9W;zueitPS*~F{*qF{9t-Z)h z){TkVZ}#7B>`v~9e-jgT7rs~9zx;jJuf~5p{;aEv-ACL|KSNk(K6;VNt;WrZ{}TQ` zVZGur{LkUf`b;x4O(UlW7ett7=EpY@=d+j-zjo7pm+-jz-|b8z&lF7f@QcL1KeRrB zXT`4NTEG7sG~w19O1uGIKjYl3#D(2|X0^XeoH>ghZ2KnxrxL+=5&~S7G>55L_ zLEMA&_Q*G^z2jQ@VdnH9Vz*9iCiIoCOf!^L%&{bBs??$3Lp zgKHk;`WFdfB)mT^z6Wg;b_05xm^~t-J35rCCpGOyZvwmaA?DtLD8IxvKp8XkG57yV z+GyT)WIN$3tZChfKhM(19G2gH=5hc1&>2`uIG)|Pp6hfxR~l)+Gx+n=|Bn8;z;5Ol z%LtF(9N)A54E#9;oO28$=NL3zs80->Nx6;ZJ^uAIM$URX!*ic`i1PnS*bLVZmN=Vy zgyo$t(EJAtK@pz3+~{Wjl%aO z@*MUv*U|UkK4)$utqZt+2KNsasi0o0K0&(2VxJT#A?(;o=r8`9vp%WkwD>nA5l;+y z0r+Wizs`rTmE0HmMc47%yj#<0(%=N~?*BSoe>wC2m)XcOoFOmQ6RwAER2zhD;eA(2 zuUySJsO)jV+>}cEd%52Gb@(N|r~TB0{=CaT`tG<|v?!{KANhu4`s>KS$glH@J@#{F zBQLZQS-4-wSJLl7{_);=zneb~`5ydvwBWB#|HAY0JuO{9JaHh}O|3$X|T8^O0GQ)lt`qVc1)^=pu zGWx0XxgF3k$vii7#i6?b-OvNQyV0M6e(?Vbb8tWUH82b#FuEK49B_jN{{7#sf9v7- zNP{DA6pq0JOu`gQ!wk&A*$qnOkqdAd&cPxyF5V`l<=bQlb9tGPWuyak!Y;ogkKJjf zWEV0SM*D6P7wpEo2T~vvb`y3lav$u$oC2w^57HnV_Cp3_LKb9$4)kDva;OG>oohs< z5!YGF=|~fH3)mn4doYwgUp6d4BL5CM^3V-9H#^8*g#5pY5cW4SXg z?NCFQXRH3EX6s&6b5(y=b8COEW~+wPY?(>T<+iA~mH8@y`$fW;(E)X4dyz76&D+Fx zW<#wy^Y+_H`>v8S>?@gzb03|SHAsrP=1L&RHP^z8a5HR&Z^L)t9(Vv|HP5QE^d0Bc zKc%#ql`?C-M9r>T$Nkr+xht+w4-@9rK<;zSD1CeW|(&8O^iB09`cA zgAj{34ro?1Prw1tLIIRO71RUOndT=DPuK)V$NV_=O2nK5RF4{}1`XAa<~gW?C!qqK z2C4zgNzkwco}Eg&yodfT?NM$fIvJ3MIUfpAE)u?{$qF> zDniyvCHuCk&awUuHBbw6leB?QpRisUkd3VCo1iI+wmxz_bv1Jo(2&R+1(+~5!baY~ z-=8a|KT|e&kyJ)g#-SR&DyWTQjsx`Pc*cc`qzGBKbdebP(AC0T>bOWsT)Z3Zn$x@+ zQj5QCl6S;EADK7pKRdyHh8ktlM;QelW-p9W7mPt4=3W@UuOEgm55kCvG7kQK2e$Vy zH8^ zBENtr%&VXYvj&ENCQWl5@_?pL(}AShp(#dQhP)a=_@+N%F6?FIGjdnZwgpqJU^CnT z-+_DK?2Y_G7MTh^g$kIv`{}?2vrZo?qjUChjHuAxo$sY z@;Szr#f(9ld9HSn+|9jSWSwS$`@hb7*I)UrJ?O1;^DH*eEKv^i%q3co^NWD{kfK89ZkY{z^fl$$S- zPq;S1ABH$H@S7lgdU==GlRE-uAH6bg?k`&d`Ttko?D|gwXYYJ1aPH0z;dg-pgc*i2 z&wdnm=J8JhU6|+n_V2zGc&08oaCWdLaQ2Scz`27hfpc}3wgC(Lo;ThxzBOp#8f< z^rx?s!uhMDXk(-lCq#+?b4l7-DV@4f%5+yrc{1mn@4Q4RS+lJ|R%5S0)@Jb!KBN(Q zJ+dL~D*k(WiI|`Xn!$YJ60txFw1U-jvDm=wxLDdIIG=x!{|_#+{~y_jy=$8N|ImGm z{r}MGV*fw%r?LMZ2KTc6ABGcH|Ao;o_Wy&Me&GL2KmIe);_vV!EQ3&7H^6KV-#hl# zU=8+L;a>PT9EKX8cxgU|O9*=@-+0nW>Sx%2Ibk#NDbNZpf$LhO|N=29rbuN*4qFIC3U|B0vn zLtS1&onDKqOJ}T>@h|-+WFvM{JmWORdCf@k1bTW?=s!`r8Yr>Wzp zH~o9iJE1E9-D$Y+ACE-p#+}rW$zPu>I*>};i2XnR7s;Y--9`TgF6{ryLO*&R`hq-z z{|r?P@9zlb8w=pu;v4M%C*Nro^jO&EfTEiv<6<%Iy;xdLUo0(rleYPb#R_eFTlSQd z;v(EAgA@k5@P;vU1^hrM??l)Vk1(vN)zIf(xVav1w|!rcu&gnsZsIauHX zWSr4AZc|@GbZO{EoWdy&&3#VAg|R-9^*`@Br}1X>y?f<_O17Qxi_u>)=~(KkS8P zARp>Mh~pBt24dk3`0+~0EgT^pmfJOvA^!Seo!o-F9`{&?^M}FjFSI=`Z=)YiTjN3Q zq+ig;IHuY&6d69R7&?qm2fZm3~s!U>m$+=Kil z;qA!s&Eyqf-obqfG{U10&h?++K96}lG{XmQIUIu)_!zE&<6wus!!n>MpeaPAAOiul z1x*s{g4^H<_=31UBTOvj)$lI%SFx8sD%Who?GpF^_bJ@U;YYYTkv~Sx-F+#2XZT{B z%#Fgwn9cADc=l2`^L!|C2Jj5#2cRAP0;Ktw4eNxZFb%)w(X1gpR9iHsFr%iW*@$@s zem_I5Mjk-kiu@*YU>*ikSr~V$<$nL-nK0%|Ft3Dn;j^`J#tU!5ao7%P;g|p7KEmYk z%pbru%=pdy2=kp_g+ZS2CGg8SI0%o!eXt%rSWDe%tQyFEUZ` zkp=ueT_2yw_%~6Ckj2;yxR>BxiY!Y{l=6&3smMx{N@NxGYTRq^uSM1o-WZ=q-#Sqm zkd4?)xHsY7j5ME46w6|wv@9n|E7FSHhPxgAHe@^D9r5fV=lV`$7j`G^-T3z)d(-*9 zdj|h^&*J~?$U*EwxDSUV$_R|cC5j8&-~n$K|DA_17zYjichBbCwY+;y=nlyZ-yyol z9g>I4$6gS>L-dI|q!3wzy%=``{w2s#>}44{q@1$^Dv*`ft8lN5*da9`JERus5_X6Y z>M=J!BYq}m!rTnzr5$2{7R;?+#m@$I%x%z~utOZsfw>d9@N+^p<{s$H-6@7o? z2=>to##mYCKqEcay}0}EA485~?;`J7+N(N?zM`k{`+B2hxn) z5=?%P*R4peiS)FQo?JhU9L4U^k=_Q<8#y>be&9Zg{|K@dd*4y=Yl8ej7Vo=V4C%K^ z39>Znb}86-JNg#4OChoddoJ!e{PU3c*fqho3;LR(Mdn~{Ox-S~dA_I9d{4+a?8cqj zSwGw^4ah3jcGeHKvwpap^~3E_#&zXfUxBO)NRa2LgHFL)a25_wKSfbrG9|5Hn5=mS zvkCb}@WG4JZGWL{`YmnID-eQxE8GP?hUcIh-k^Ou3n7G!f_S(aeh5!PHdMivfEt|U z3fKa-!A`gz9)TZ08dO6Qr1Fdpuc1wV+ewbDRkSe|a~Yw&vk_bS@r=dkBtzmj+^=2^yw zdr)(w4CO}1aKU>rQvB|Px!k@g{ue4&$Ux&&=<9wgKI<9Qbv}`C>uDL4x5dR_B5n@G z@o-p^^?4DStQPxAte3M1;vM2&!8-(VeY z{~Px{!hFL0in!)d;_)kdW`j0itK=**{+Va|bDHrFGC%7kDM-3W^c$5FX0rb&ntuzj zm(YML!L2lsHH>}eKQnJ!fvn6zzZ+SNy#`r}KkEF9y_uv8U5!B_9tec|H>RrC;Xr1L;UA_2gbJ-{x6joCI8<4J8~NR zmnrrq&agib`JeM=nG+f#u5qxCJ_hpPJZbe)()8P;qnq@3557SfU4z>WWH4q%+*k{#2g{iVKzcNehtuw*#u24^q;`Ii#(`i zZf+-YbUcFG&LJeLM%smwhA_nZ#UPBpFu1}=L-2qb{Qoxeg8$!!V_;!iZ7$~= zJx3_}L^(IX6^agfsI+0XBOUm+<9^{k#7Ji*b@G1JZqk^aMV3b}J`ZDj9?SR~S)IuE z9BQ%GfH9KsJv5{=lHvcuZQ@0dA7a`^dGaR|B-{(hj1Upe*`&7cvn32Ki7MZ-szjf z$NP+h4a%}!3-9#L)#MV9MT5$?1gUNo^?p0?iy**Un9-X zh~0#}BAETdi|nHcp{y7Zq^9p0#+8gSSR=0vxLOK*%u6OQPpM;`(!{(Z=F(lvQyxnY z1M8jIaP}M1&T4{{WFzw?XxIB_=dtUKQtp?~mtjJ@57|4*o(UAnhr_4`BB%$jd*Uhg z;EJXm0RKP7$H4y&axb(pY1{(-Jsf6eGqHCAY~a`Z>tw>dqmMlt(Obm-FU=k!dpF91 z#fj{)pu-n{E+6+U&q$E6MRfeI*F>Q67l!U1vVnaX^$EdZj6*jN*&K>%j@9@coOW;0gCeYCR{`LybW?FTV)?o&bih_Tj8UOX8Fr0?1OUHNLfvUFv{mf zNCYFi3uj?9W%g>gn7G!#f5DHT5;~y)df*IvkMjEi_#xy#2xU1Eo}m0a40&)F@Ai&g zQqG=)L(l@hhnL|)&=@HH)s%k|eQ=Bh<%UEe*uin0U3A9F%|JYXSIe z|M$`UXVLy6ZP@MSBBhObrrpB)H~jzpe@8ssPTCL~^WS?Tq>nlB{%OiT{no)nzFqpU z!^jcrqem(KlYGN_BiZN6H%mBQ;^i`Sly7-5K)xgm){suC{NDi#!+UTI`FKCv%s22D zyaJbz?@{m_xCS-@@FbK#D9l_&e!v2pfjRgX-iCFTlg_Y+`BPvBsL`O8mJR>FoWr_$ z?hxmr5+<(?ePJjlN1qr9p$Ll4vHlJvPzq(`=>J!v|IZ{rCGk}0Sbt~YpayC&*ST1K zXX2oqcpLgyf9D=1^y2*baLrs}-n&99`&Mu^JO2VsWB*?U`_J~T5IeMCZr{cEz{#A! zzmxNUli2f>%HFRO&IiW52lw7M&i9SyeBVU&pCxeiPc-NIZsdGH7{NRmfo@$SXA_09 z|16CCXQAvr3qcp}96Cg_E1D4UKa}zxN&ZKW|I;Em^o#PQIFAS&t^&g830Jr*?0?{V z#RbmspzSG{XHV-a?UIT8zeaSwoap~L(ElBxUFs9sDUn*tb$WEr4Cry`L>jU=|AO!) z!Z+dGjJr8Q#Im2gPFd`4%@nbwJ)msq4=DRy_PL_>)Skkg&s4_1dqg^SbIwRI|Hs+I z|8Wv2{|S`;m;sD9KQt@gJoy-kTm=y@{rH3G8^~ygg6rT0xD{@KRtxvG@eJiW zOEvf8ne9Bg&BwF2xF^rp!LvGe_IBQ(2f8skaqGfwU>UuwCC=DBhfdofdRs>HwoK@8 z!60Uj6TK}5dR*LN6fT@cL%rtz_qaEWGd+2RN8nDNTGHGN_rvWledBI*FY+PCT37UYev4r)_Kd(x^JS(Qqf?MF+-5N$NhR|=o=&lN&imt{ZrWwfE>a;jQj}yb_;F$=_KiJ($?E((|xq< z$ZqUCNV9{!L3t8uhj)tq{|Z+4_x}~zF#nS{uHd;Jg;Zc!U1Qo4EQgWJDYVtdN@()S z;D7B0K-PrPS3r*9{zo_i8^fqW;U-9g`#`gwz5!?<2Xc4QH-Nmm|AwEyFW_0I zfLdq$GV24EEa2q)lUxG0>pO1Kfe3%lT9_zj?csTqXl;V4YNC$N(5=5n|d5+Dig z0)7v#c>**}%0KO?)<^kA=3>`PQUA|S|04@d)BZ0~|1Z=2Ba5*ca4*5X6j?_2@*(Pf zuCGK^VXxkU{$DEkf5^IY^#5ox>a)=QLpEYJ;ogLQGt%s23^|1UpO3yI(u&2+5ZyE{udYRKa3`@ z|Ha_%!?OP+jQuXzlsy>Br2h}Oq(K=pLMOZg6YvT7|4%rGnc}V~0;&g1Gf>#!T=`5w7{15%R~ zB-xc!W%?j=41(zY21zAU z^##%X1u_4{`j0b6>Z*gp$YOPU8tXsMn92GNG@<|2%pPcSDC<9w=vG9q{<9O^ieT1% zlF|Q(XZ>f0Jq}^4|3C+H#4Hcvk{U6*bP5!bgJG-9tuHsIfgG+}Q-HtT3}vuJy>`QD%v ztY8B>v_U)l7Ke`S&A|7D>@x8^TKFEJ2YNH;|3E+er2!rL|0jZFXp#N@%Zz1@QT~_s z-oQ;;iu&(7_1}5wzw^|8=f#+IUg|T?GnPFsjhIc)lyY90!E8G(7Uy~T zAm?cx&(l7h7aL|fZf(eR<9X)L&P&I_dFecTUb+^~i}T!h&hR_W`JLyb*Kl6?s?SS5 zVFqAOe_r^%JoVps8S$N$QQdj!zw?~WcV0ZE^WsJNvd_yHjDvlee*p#4?>w>+-AKmr zoF~x=X2#Q&p_S4dKtB_@=w~{iBawb+H2qSfBc6U{7X3`Fzwlpce?A=J9EN@LOUaKu zxbPoruJ>}S58U)iJuu4kF6?>qS@R(`mVJ6iZ7AO%@oK{OE{V5_pnD87Yk@W3G4Kz?5V-++hV`;ZF9;2V4g`N-db6*?giw+G-^I1G8v2446b zprWAp9L~Uf)XQ0r3l~$zN5G}Pk#d@7WE>oX9H@jJLK-xK9iD`HX%8NRzhV9orr~Ya zgxl573s3W2eb51K!5gracME}1cmg!Eh1sBm9LP1%{_AM}^|b#`K%1_oZ7)O?8EF5j zY5xgRf-J>eKpfxX{VSGf{}*ZhPt*S6S3{UuWSxWYAJn^;|AR)%Ci)0XHpc&iG4Esk zKmGqQ{>Qy-hCcT^eQz7(-%0sLcKIm($p4K0XJ~KcX>X{{2atorKeWexzU05g|5KEI z!nlzh;_y!VYyO{j#)kss1kYXq-z8uEN&0Um4Xa5zJ&b@0dlBvdm|rH$iS6QQV$Hew%pT$88Bip6MGN1He!c4ipp<8_~sx9_cn+Jru8+Euo0pm24W!&{>(Lh zg-PPnB$NNU$$!YfoSV%0-$v^Ho#cO3fV@GNmO$PQX1L}x>{H<5Su&9;crU6yO+J*v zYU~ZjE08wiW@Ha?8`6c0NB)Vv&K}aC4jSMF-uYoL!*TckK7zkNC~2`4eho)pn?Jpv z7Mh`#@2HROsUPY856mEP2>UQ{B$fH!ea!!)GtZ0k{|nrU{4>|x&U2)JCfFs}v*@Nn z4&*{R?=}GW3+T2_qkq5DCxwfB=(zXM|LqgQqQCt|$9);yc>K$u0{=$htI}7f>T~FS z5T+LD@H0;JOZ`m0G|+EobkY7p6Xxb@^gk?&|Dk1LzqHz@hiMaSj4SO)12RQ??X;DS zYU*7R^)B*$zMYrhHq!0ed^f$z)U!J3UBdkw`=F6}6FE%S5!^Q4z-;0jP>a|7D8z=e=uZcz@E<2=%Aw|B#-Iq=|`iYa)%BF`Gl^|3C}@B?{|DSWt0#&6Pb&Q%%wxFu^40gXa3$1!TL9{GmLX_LfPAx$zH$=_Qr;v zmEN?o(uaFL@eB}dka>OT1sRTIE+U%oeFSq4vwZ)j`Tpnm{+9ye6yMbhj4$S?HrB!3 z!Tdh_iSjuO1C;eyK8-Bulz&7~CQs_r@wavApO~{_D8rj+TfhKb(#8j4yz{uxi!N}m zxWKKazXgM&!O&En4A1n*2w{5r`lXMw>px0Afplvoju+u~*HI_IIViYRG&=5+1~K@> z!t_tfivJL3rXS!NMRN3zrV#SsW9q>(Jlki;x_zW~SigLQd3u{!RV9?G>SXd`C;5>| zex;FLP?17@fo9n&*;9>5yU!~*N4=6;?iHQMD|t4rF=tA$> z3Eh}`pf?5mf77J&S5MOaeO(6oCS?eQr%%cVj4qty{F{^FUOFir@M8ACnCqmBgJ##1 zWP>(kN^&3VvAOiN$*G~fTU3^?Cp{6Ee6zsdUl zUu0xvMn>mn#I-mh?&TTU{~7V}^FN<$27R>|ImA;s>^mj3>8GSB;}m)@r=&jh6#u_mmKE%0t54ffLRFE`$_Gc--0==7C|173wf@Ca` zdA}dg7iRx^F&oAVT|u(Ub-u+y)xL3D9I@ll0iB8C)UV^>+%qoSyT_Sd9v5@OO0h5* z+CsS2xB;;e#uhy;_Rs-oLyj&Ch->1HoEI`69#@EX>1+EqRCg?CHS4IH&z3Mw`{YAE zbDZ^=Kcb&G&iu@{44)og+%V4g(#QDHCxs{^7e)J|7=>$tkMs4S$D~xhTGRq{PH2c| zZT|TH&LzuUDLTkAaz3SVwbbqPi4p44Ls(xnsYVtCOrc)RDqw%j{tzh(ACvO1W>pd2 ztSUpBRaMBCR71^vpVWeOG)QV`cU>=%hL57lbRW7*Z1Bn@jd~L~>yrDg0IDhpL!QOo z9mbe%GwJv)b|K25yEOp!_zQ>+~eYi(~|DvjvCHPfkTLhs#pHbUw?yGk0qxLoRGJ>w(#;h^F9x`=;b6@7}; zj15AWYl5fX0H8LZIS7YA3*}G?^({P$M z=%1`*j`>aGsN5*7;;rH?xJlf!J9&+l%kgKahwEZR_xR;9g4@ebAm}07xL!^Np({hR z{a3h>=KD@~1fGE+*v_lm0e8Z8;cmDOtn~dJK(=JC&PX57O#j6K-u-Uzl@H4pa@;m7 zqvuA&mEjh*@fGnP2lu*VDBUf?nQrEjRO9ZuGHU5ySkjltk34Qj1#( zK%X=$g~+0eVaW{`WjyDWytG%C&vMh>aZ5JOp>?<=$A&$d{%@#9wBaL?6W}4dM|2xU zBoCRN=#hdDkLWQMMtGQS^GNZ2j~KE%Qj+G8(tRE&v$&)@+9MUm{v?&is*N7%{}HJ{ z)-H}n-NK*5xO0T@UxR8Oo<^4k{SOcQKaVsY@rXIhL;tTzwe0dptD#m|xu0#QR@oCg z?9m)y{ii`Wc)pIK#BmG!T7`61t}&S#Z86YXDk zfhsWRl-^=x{o5)<)mACqSfC7X1*!yDdfH0+XP5G1t5oc1m&zooR6+ILcBw(uW?7|f zmsO0I>r<@Ku*WKmaaJ)UTBRw*D$UVWG4HjDW!@?+At$62X`Qu-EudZO$hP=)X^%O< z{8u~rZ@H@Th*i2Wt^A+CD&54>6JTZkyH)yj6{-FZs|-vOs6p;Gv{;~q2{&@Ao&I00 za`Aj_+H%i)f$}EnlyAS4{;yTWcUmc+dX*iZS6XDwjE(+Jo9H5K!Z8<`{4g8!vrY72 z4(9)C@+fYPg8}!F5Sx?++n8^)p-*j-iU^xjhTE9`Y?B`mre=B){U4ju5!QIlA@#_H zqYi0Ynv|d5-{jD#X57rXZ0M8Qq=k4|<7{F@+9C>-J;jDDiC(oQ*m!@v>fnBzE{AmO z(kUm;*PS>iJwtlco0Y5jl5EnSX+!_VhW?*k4NdCRFmiM&v(t{QWC!z#tWm`B4e`CyI}4T3SE%Zc`B`?! z<9@nb+%woNIX=5+gY#7OQU`sh*v9y;O=>3Fq;{6L&b6`rU!dw8ZPJilq#766SpO+fP19}Y@O6qg zGhbO^+t^R=nzZVQlr=U_*}~ey&U3Xzx3RaoNI5J;ssq`X)y6!eO`N_q>9%!BPhT75 zzm2umPU%MuoNklBr8XHl*CxZGjb+jQUo2J{pGIY8TSV)! zu>NP^yEQ0Xm_g+s^Cv8{C#|B7wMb#Sll{*Y>OYGZ0-REUERD1Nm<)TKG9d2fp}IZKquSE3q`IceR@U9?DcYB!zK7RiUaV6D(w zO2J<8KfM+G|5o}Ftx`Nm{_kl;|N9LoMV1k^e6olBNUK!N_i+B{8&aLoBQ^RQRf}7l zgZ!UwmHOFMX_#zf{MRa`rDE0OYGq!&RGA}M$^TN-!u?u}J?MYrC|hDH``_M>w(3&V z9;sE1jjhr_xX$oabOuY6v#(TjBYSwR-Xsh239ZsU)r09w9x^|}ECp$&M1Pd}YZ?8WxZg8>iT(}~za>oKmL3!8e~V-% zuNUn`=6`2LRc=Cu=pZl5r}B5NXMUgg|EW<`I6?W3^{C>#>%}l_P$lX7|7PEMDa&O2 z-!-Z#cKKB0PLHYzT`$#4^43H!|L+)8b*UX<4B#I<#NQCSp8l>+nRKJ7Dbu5x^`pw1 z?NgS_QPsjBSgqCTDgWJK)2|o1k@_GnycA^UizWB?< zn>4OGF_*Jfa8$VhM%5_uaU-XlGK?IGyqvj(%Vhxip$~eQqw7f-Ro%(V$)#P6ZW42J z$(J)%>l8=Q<cid?V_dh??JH5WLoHGh+>K79R#g$MvQMijGP)T9WUI2+ZuZP|OG$D!dosGEm~chD zZYc~dQTipy`<^!?e;4(Eshc^S7Rg2C7^yEZ-jr;lCbCP8x96&f`|{NB4KAch9lx_g z-NpBIyiUh|GIBZ3K1WSFkfV;9k;TYu8vfUz;oly}T#c%~g`p$w7st1%ryr0EkHG&x zC1ijJ_QGK}ewO>>a-RoA)kK#^X(l-Tk#A8uBa(y6#jaaK|BUZ7A6bC?5x(^!++zxz z$qASN7h~0pl$El5jK$MgCn~sJ;;|>fPS^#>&?4J7WBnE}f51N5kM9us+1sS;v+Kkv zw~Fm63K~j4+4N7esbP|{Crol}UO-@MXJ3Rr!#Q?|)2Hx_qi?_o%9N_*BiV)l$oKb&gIkW`?o;&%8`>7~g-8n9|2o z6ZU45kIarRu|SI}Oj^OJ4-#8Sr`TOV(xw|z?Y=SPFoj9Syi0YKhe_ADFxG#=q`RE{ z|6-W*E(A$mj8FAb=MBK%^r#v#hSC2Cl95=S8Vzno-(pO;iQ5xE|KAqI{7e}8UxH*j zz^60;F8+TpuCy%3msjOQcT0ZeTIvtxzmqwCbE8WY&3ILD+_*9v@v4#p zmnub;p`%fb;#Ng=hg9xfD^*-yjWSFPvUcaVs@uh(hX!Z_6Ey8*?zgW)%*+p4 zOyjDhdR(;*trhFcsIocMihX)5`af%>T~Gi2^jhi2M!zh{tGf7ZozUIqQa$I^O7A)J z)8f6VKi0wcXIu?tua%($`u~QtGNM~6qtnbsM|hQc!KFO&)4~VPC9iC+IiL{cJYj?JM^GT(9QE&lGd@t$=b$#uWCWs_N)^tvJGhmV+LuY^Qs19 zy}_+a`gQ0?FgL1O#~xval!fO#wUu4ts%FTo^u~4QsB$)H z{J1JyR)@4^(*I8` zQnrjD^glbK4ccQI^#4fP5C{9e8UO9<5NA@6>fTqRdQyv2FKtyH^e2)3Fu30#LogiM zAtNzG)Zsg?&sPE7h32Y2eo6eXox zQ%1P*g(#^QijvC7D5+YCV*D2+H8W9CI}s&y*p1lh@oO+fiGzFh!7TUoA*sIEI}*kC zKT297qU0!Uwv=eGCqzkGSd_GfbItx}nZUg>E?T+*qS1efmhQ+X=0Bq7|3}exkCOiQ zC>hutC4-sl|J)NL!>Lg+k{l(Y*j?D&_<16tWYK?bI1T4OW7tanX{%_R?0@xbW&Lj} z>wjBW|Jf?}Gh3x#eyiwDZ{_@_t+GV;0NS}=2!(JcE9c*WPyv-t)wflup=M|+_1`U= zO>>JF_uRty54T7|#4XZjyoK-o7RvTEY2LMs@!vMF9NWhDk9LX;W7dgnV&fot`}{WM zf3`{cP@Fhianf-*PCEPQrE9WAoPD=T_t5PU!To!as_6gL(EqKc|64=w58v4IAa@3#xSsSD!dxOLfpVhfVVvz~ZR=z<} zk&bg)`2WHd?!QI4vbTs6x=kCThiiIW8>DYxgY*w=kiCQ*T-v~T-v-wEHps}t2HA<5 z3wtusW8A=617!^|AQLnRTO@mVgJ=UbNKV8CNykl>vPJR|Hn0w}K?=e*py#uN^xs1N zbqn(-TNr9}q5qmv;4G|lt zTW=Cm@az14;U;MYjdO=&5A9$KwS)05=U-sgxn4s5o%1gyIR9di^DmY;|HAkZb0?fv z5b!PT|1Bvoyd zjY!k#>=v$Fc_M4G0z=jKMhge{Pr^ zkKR?{*LqhM^spT0Ww9obzi*23Ptbp0{k;&1SdTAWIwpop^dC~tf1tf9OFKqil=c5a z+9>3O_5Y)+|EK;x^nW>bq$ve`OfWmpH-#1#`hQ@>YyAfW9eoETR9ijQ$Tg zK2CIfx{*EUzmr~c0Q=xS|IfxeH1VDcqw6zbL;vsCd#p#J|C5IP4|p@t|A8^gMZ(>Iyc6z&FR?#_ zd=$P(U)Dj`Utng*Lz4?ZA5xDXLy(QgIxs^o41$yJFCve_2{;LTxUC`l5c2QX{{jDm zDcYYaaDOwN@gnzf;r}h{p}6nB{yp&F_Cw_VK*z?<`0p?0e%LtoW8!`3-gy?iO3qo? zb(T4b&!lDFJJL$oFuOSS1MJnD17hM_5W?G#?vOj=e~D)l>4M*2&WEGi;|TQ6aLz|C z=hP&hkv_`QaMoGoSl*=%#W^);cSt19R!aC*XoC^>Bm55j0NSB{$e)m!Wy&dKaSrqQ z@K0F5IJn-&xiZTWQinA19!)8n0|Jd;f{JX;rAb?0ocgYm_i-)__S)>vSf_Yjs*OA3 za^koFZsa+(Ag_X3;9Ky0h{OF6R_agrbj`3dqjrFZ>uLaLY&DM)(2bB22<4vAqr)-WY+s7+4`RJ%=>1fz^p5G+F`D z2(Y#GAPF(j%;*AfgyrbE@B6;*>Z7{4yY9PcnW|EE&q9rc4y2hLBhO6NnThORW7!Dp z25Wzww_2<>?5Ovb`5gIV<~N_`ec$KF{N-C`*2l%yX`6n}_W%1#Ux)ZR*1sZN{UQD+ zTKBxZL7SGhX!{BNc)I!L`|hv*jK7q>o)+u)>uK4(6}o(8eEtr9{;U2!#g+ZIw$Gpc z=l(y1_{R5)?;G$N`$hlXLOe0IACrIcqcKI(bcuSO@%xC>|HpnGkp?Xf(ePin2bNFk z_foWd+3zcU-R~>@f!|l;pXjo`oA05AY8_KY^*{7u(Ljwq@ng|M%|G&I(efid=1bDf0pm2=Ih3X z`E07h{l?%2{@m|ev>E5Bx);ONN30M2C7+MD{YU&R2JcR*xXm%$;-0_tI~n3G>sR$H z%)Fn^M||ZmzMfC%Q?PzbJo^d0pig}xCVqfV$i6YhOk2LR|4K~q{7ljM6MjRTR%wm4 zUv=K7G|y2yFW&{_+!W8rWBDiaY0%<7)TeRwLL}G0zoa+m-_eh{&-(}TQ}omHhpv$y z7XK2xhra8Y`gQUDYWw5juTYugYWkYxdMc;Ck*--hO#L)XyRN-uaf7rkh`Yq!68~%J zxBSEU5`L6MEKkzUS$-G&t-O9o{5|ng;@_sa}eHHS%ON zQL{E|`TC2|`s8=?e>>0A@!da-&aXSKpLL#p@}Eb~Gp>u@;=_3Oh3Nl;>*VKM7tXx4#Mc3x?mCTQ|+oL8ErOLX~V*9Bdr8M^kc>w>P+4Z8Vj&MVE+0xf>od8K7q zq19h=UC{dX{`c5Whs`I=YgdPDb=WE5?|Rnt+_)*;@|?MThfj`H_y0w#iI;6(v42s# zWdD28w>}{r{T|=n z&|k%{c!UONi27-Oda3x|38i@y;?6JsRdk8l9or$VR&krWn#Cw^yBoO{+xSzdd>Eq zw*C{AU$=ZjiMnn3hqjH2r|JK)?O)Nh<)5_ON7lu^u>39hN7nzH^?zykXD$Dr&-{O5 z`G>{-iSOi}692sIZxcUGAE1Y=|G0hsB|T>OzZU;9+kaC0dGXiir!D_uYPbA$aliPV zB=_yo&s$!mcUxW;|L5)t9+T!>^nPhROy9BnU+ZJO_~-n;f64#*SNea&mDa2NmjCHm31|Ib7Ee;(5R^N{`@|D&Bop8h{#bnBrQ+keRY??e0$59$ATD5k{I)-Q=K|J_4z zMSRuzjQE=Uv*PR4Z-{R?e(wGLzx|UB`JD#;-~R9J#L{m)p6`BK{152AqMxE))@DB~ei!``{T#iAo}$O; z!}L-5b$Xw)AEgf!()M4m?Q8VE(eKz+F8&SillJ`@{ifwl(I+gA+jo_&(>g8F3$#t& zpcm;c=(nW3C;kq7%JTmy{+vAio%jcx&!=tsQE`{g#b<5*GJV#zXXstFf6aQkxSOui z#U(_UEukA+rr@j{*^68x7 z|DWam7x#Du=>4quR?nGlB_8;?`G4Xe>%-!aCwBG!x&NnenxILVqG`HBm)~&zPk%rE z&+;r?ryKPDd;dRXJ{qmi>eGB#v`!neNn5l%wHrHQyRlxhfj0SzGwp2o1v;Q_TfZl+ zvagSv=Pq@#w{&maq;tSSS#2*%y+4kGw-=*j2YgA?1rv1~F=c&$m6D?c*Q`_4te?$B) z=$q7K{a@SnZ|Dd8&cF}TJ=Ro>{=I)2y{?Hq`}(CR z{r~nm|2B%>Z+gG{T70)&bdOO+IdqivcG}um z+S*y#+F9Cq$=1uX5N|2g|6D1r%H!3>La(+7n+lbdnk_Y#mRc;el$Kg8wU(BOv~926 z6t>4(?w9u!L4W*Y{G`_}$4|w76aQ`eqxk9g?-=HO=HZ{Ew--Ta=|ZveND(|*1dkQL z<3;d}B6w#JJRvBOT_}z!k`^h8ghjd{S&^zp^lo|&y_ft>;Lp6DK0v=hAEXb_hv`@8 zBlIMFl>RAwjDC%No&0d{&wQLdK~K>q={M=O=(p)<`W^Z-eTF_uze}H^-=k;f^YjIJ zmY$>M>5KFw`Z9fmzDmDOe?VWOKcqjRKc+vSH!3fd!tzpBQ3|U{VRb32DTQ^Vu%Q$- zmcr&z*is5xOJQ3nY!@D`7I_}77P%d+7RMg0>FoWP!}a3W!}ZQ0+?XiB&9YKh9D7tX zQwnE`uzs-!n~J?hO-1^n)}|tC+bqI?fg&9A4|RU#Xm(n7+%r{#gSDlwDDimkauJSR zEy9~cZYO0${wH;7ML1BDelodTgi}S`PNs@-PiuMHw&&@ z-dltXlSMdG)F)pm%FQ>6dj56!wIb}QEW+N&zaD+C@ZEvZ!f?@6e?3tg_18Cx8vgZK zQMd0_6ghvlv1p&~cFF%|+?~_LK6p zcsL$@Ydrk6c=+vJMWiK)aIjWbN+3nC|KZY(hf6yiF73Dw4_~n3LTSf^(vAzI9goDr zkJ#}@X~!d_O^?RIkJ|KTY15;nO^?OHkJs-bdq+$KsL4-weP=xSL_Au`phWa&acGI^v3TsQ z;k@dCf6V^JOW`}C1o~K!s&u9wkH_B{PIU?U@nUo7Y`-Jk@z!v<-%+YmsVUx3>@1!4 zcg8#48cuwvVcuD6E}i=);)%D0lV574CyLFb^Z%}R*Smf({`>eJ;uphDK>s5BQv7oK z@=wMu|8o43cz3+}6Y=ghYrCF{_r!a?9Pf?yzAxVU#ka+KZTrIK;(hVH56Am{JKpyv z@xJ{n+K%_f2jT}K6_?7p>uYB-N;#WTQbo|O6#|PtsABzt@6(9Uu*!ZFNQ1PM< z$A>-ar79aayeC%US(o^xVUyqM{{QV*Iuf?y& zuYWauBYxw3@f%OZZ&>(Rd^|q>{*T1RW%%(Y;}h{zJoO>3k_092PsS(T8=w5`_~h@z zZ^mza$m>J#Tk+fRKgRzQpNfAL|8x8=@pL@>w&&g!PyfQF>8tVdi`(%#@jFj{B|aUW z{)I2nAH=7hel|XB^)vD5=e#b^Bk`H|%zNT9PseATjn6#ib%AXCllY95{A_&osrc+y zJ{P|mzgxVXp=YCrgwMt2;&Zb6z4$$BzxQ-J6VDVc`g}a|d_42Dc;=7d^YMlF!n@-O z_`V?77oLhQd@{aJRK>~rg77)73-m~Q;rHVUTJqU=_Ve-VbMfr+@$45r7tj7tJgf1a z3#lY}?%na+Q}Ns<e#J`-P+i#i{1LzZHV2**lmj4=GbkC z-L}|mkKK;g?Tp>7*zJzpzS!-L-GSI0jNPHw9gW?w*d34EiP)Wt-Kp5U6uXyW_e$(u zjoq2py%xK(v3ossZ^Z7+*qw{r`Pf~E-No2lirwYdU5nlI*xiWT&Dh|KeytFbo|d)H!b zHukQ^-i_G18GCcFw-|d%v9}z1E3vm0d+V{c5qq1lw-tNav0oAUm9bwH`_-{u6Z^HX z-w^wavELN?&9UDS`>nCx7W*Bs-x>Q|v2TE|C-!?|zc2RtV}Bs_2V;LI_J?DCB=$#R ze=PRLV}By{Cu4sq_Akf&mDs-;`!lhBE%s+)|9b4-i2a+fKNtJ+vA+=ei?P2H`^&+h zzrPy$Yq7r`ocQ~j;SKo1Rjn??aXnbQVzA+QuxDnsD8Mm6^R&(5HiCe94t1WJ| z$E}XI)fuWN#uajP$G^~bG&xHTBJhU3;q+!~ErV{vOdZcW6k>9}<%Ze5OB zSK`*yxHS{EuEnj{xOF{l-H2O?Sd3dsacebht;MbNxU~_tHsjV-+}e&?J8}DB+%AjT z<#D?rZdbSpw`<~dZQQPl+x2m~A#OLu?WVZh9JgEIc5B>ji`(sSyCZIQ#_g`S z-5s}k;&yM`?u*;~aeE+c56A71xIG@Xr{ngexP3WpUy0jSS2#O;;1(;9c$;!b5MyFai=ft48)zmxHA%W#^TOI+?k3y)4}|I=StkU z8h2*m&TQPd5qECJow>L(A9ohw&SKnIiaV=uXDv*Hc(E*AERPo};>Egnu|8gGh!-2< z#nyPSEnaMo7rWxcp?Gm5UR;Y8H=hrWH=b=g8VNe$pgIm}f@j>4w~bjB@SBS zpe+vC<3NJWIOvLl?l|a)gWfpki-Z0+7>I+xI2ek9;W!wHgV8t`i-UUc&R2{s*RWG z;-&g{sUco!jF+0?rRI2PFkTvpZ(fXVmc=(K;+r+`&BpjosGNKU+!lPh)$3a;MX*`~n}Tl-#<$zN zij}r+?Zmg+y^56%uOjaBD&nqhtrdZ#?jqc)T(Z zuS~`(Q}N2Q*DIHb;93#P7Qyu*m@9&L!K+n8P+bHyMNnG=bwyAwc&*s+TCwA`V#jN( zMbH&RmX}_wc;#}D&!tx@{Yi7lBfVfTUb*a5lz6!)QBdS{xhPn0zU;WmMfsPD@-G+V zUoNt^Toiq|NOC1!x$0HqA}CT7k-bG^|J5RE!Fht&^8`w`s)SPF>*t9}rPZQDv_-B(RHQgx zwuIt(QS|lVs8TbO8llt#H{z9>UV2W7+RweR@M`U=l@>}HUTZ9>6a~_PSCNOH$VF6S zzi_@>s>wo8j^KQIDf5LQ&xN9Dg3{6FP}a^jf?X)3FDmQ;Su7F>N~=ZQiv`VjRHP`1 zSS&JKENwkMLzd6ilnpaVrptO2kS?lL(rOH_@jw>QZuAMJiTK6iB zT`$rKz8^_jtQ9GWwCk2$z39CwXf6e>wiQuJDL7wlEtZQ8I8iBRk5|jQWG|R4g6pL~ z8LyT-2#S2mlpwHF?p35Lf3;HUilh{rZz}3mtd$ZLhm;pf6<)7gDgwn+6e%l;Vk%s! z0!IkG9~H@q11pLH?XUDXD3~aMsUnb1Ws#5IK~x-St#m+LRV6C{|P)QbbY|kt3_m_ms-Be7>i|SzVM?{mNVs z%nM4#%D6c8LH<7HEaR;{-zq;VHAR^A$r^7n5rpFE3EWKmb_ zkf6ReOmM!e?)B$wUtiRzzNk}uQK$MMTfz6E;>e;--n~U2!}?MVMeYvNl8#)9SJ%U& z*H^dW)t$I^G455wy~?;(9rtSDUTxfKjC)OSuQ~3u#l7CRHx~CM2#>+!X^c&#B`YYg9#uJG47;`Pa{ z*C#hc-RDuyc{F$)jh#p1=g}3(X3iIHoJVuEEXM28W3NxkV|x5Nx>1TQHN@+e8sqiL z-LGG^^K#F5Go=^=*|%_! zYN%MYZ^6DL@lp{>v(it4G(^Mk`dZWLtMl>t`poO=edp0wyuQ)&`nrvqj@i2S`u38j z{Ppb>QR8{kb{=(~M+4{4-0NEo+7f{mZL7+5Yw7Uq_EM7V!SiVBJeod_X3nGezdvUF z{LqEJ->^_@c;n)9yiqY2`n#!&%Bg}XsfwzphH9yf>ZySmsfn7Yg<7eN+NqPesGo|w zhiI5aX^h5cf-cb&nxShnOV{ZJ&C_zcQKep0%CAyhRS)%2AIYmqURCm{l2_FvO%a|d z`Bq&fc~@N}`B%xmO8!;yuabY&O`0R+R4vdVEyWwvxT|qj!Q3CF)vBGtj?>_u1dQq-KzAfSXPVe z%2-vWRpqQYhpWn3bq-gRx9S|ODs$C2Ty+jtF|R6rRh?IHuR4#b%3M|Es7Yxvjj zui;8v@UP)t!@q`q4gVVcHT-M%*YL05U&FtKe+~Z{{x$q-_}B2S z;a|hQhJOwJ8vZr>YxqqLe`5{*8vZr>Yxvjjui;8v@UP)t!@q`q z4gVVcHT-M%*YNuhp*Pm>ui;aUdZsPQ{^2R34O`MxJH*s#_ z+{C$wa}(z#&P|+~_%`uv;@iZxiEk6%Caz6fo47V{ZQ|O*wTWvJ*Cwt_T${Kyac$z- z#I=cQ6W1oLO~J@NVJV!n=ic z3-1=*Eu338w{ULZ+`_qqa|_=VzAb!P__pwE;WMG(jVMVr7QQWfTllu{ZQAkwsCE{CbzL|W822Ijcps-HnweS+t{|TZDZTUwvBBY+cvgsY}?qj zv2A19cJ8+EZ98|{7`HLHL3v{v=eF~3n7B&5KL+7Ex)b{7Sr8?Qmc7Z~B^lvsyOQ{WL(= zWt}YR zWLY;#<1|ZB*4?0+G)If1@^!1U7H`&HB*)h~zP^fTNgnmr$no`#uXlWdJR8PHz6}#} zmDI68x(4YwVRcK>(-3b?%V)Ye-kcfwGo>tz#r=zAR8F;2NA=V|jnqWV)IzP)M(xyD z=r`{Bd2}(G!}pig8y_qkH4J@!>H7Eg9@lj|*mm62^Zlj4YY&zTSATzL;KAM^_ipNm z`(Qk&ptL@9xw0yeI{qid|T2k-w%Z2W1lkyv-!kiiFiY-^3n!2v< zmtQSZ$`VSkq(lu+UX(ghDx*l@02x=DSJ=wz{fZ)~J=Y4AHWbBLQl$cQw$S~GslQKX z<-xXD)i32&lv8ZJUoloHxAq+Ne3LpqDDS#t*UwA3e!t)^%C)2(7tdF2*dV?7g3~NT zLDpr2rpZdtT>rkz2TN$q8_S6;pDWU`JKQgyr-DT*pGT#$1WS4KrR_zoLzWh4sZdd9 zsRE^y(pf1LT>!dL%3UetE2WlraOzfVD9Uo_ly4L&wOes2&zEW~6_mxbv`ssKwb1u# zS)BHYa;l(8s-kMDDO4)4sP+TG3a7rJuGG2(LPh=kie^y*HIf!8tvBC)(Bu^@wzQO* zvdFu&PzlKSow4FQe0~A{`|3Wppo`?*TNNFJ&J%Z9?7Z(XcO@3}DKfBLoUijGtz6-K zT6zn7|2iuw>@6`-DzyOVD(xwfb{D#@rkb?;(s?dDws{*@^cMM+@05-za__bKynVb4 zEBZ>USG1)!Wbr1`yrSZgD(Y;>>2X?$B^QH}R54Jfbd?m5^Iqh7zEsp@uu!RtqK-p_ zO0DKl2_D&Cl-=W!pBR+1WgtyffcnpUn(;4iBjPcrNSqQ z&!?gam*_HGp`vKzs`i8PX{Fd+9I3rKWq7`%!JMjMb5WgZ=M@wc?c<$xo^#P01+@;TB%jODT~WCiE#dk4{uWaX3&DhtrO^G#!UmuG4ZHUY(D_YYTCBqd5*23eD1L z94;x}v;CoGd+tH1qk2-7=l4U;G2JOU=dxfKmGjZgb^T_w(k&FMxTk^;?c2s$lq|u*qR8>Vy)Jc;xMbhZA zInrmt@kq0@5=Yh2SIf6rdN*lD)m_w0J*13k$5%_QPv%J9%TcvsYU-$w25Fc^$-Y{7 z)mBgqIlgv?lvk_1wHt9%S53;Pne3+Z2IV)%yJ3yw z(}=B6K8@R7uh`$)~A}v_q40P3qgEK20}Cc}?EbA^n7NHL3hbWlw6CNp+rdyw8H8Np+fZZB1S}o`o@~d8eREL7!61DNR2m$&~Cb z$^Md}F01Zkhg`A!%5)rEbsBtx99>hDYuj-&t9fQ&%-VNdRj)gh*R9`hYHrBmhGTE2 z-VJ%)(A+nizMHCcQz*5_P8b57r!W9Br$yr$6OeB>X79WBXoNqI|< zm+W8KjH6}9%jbB##xxuc*(8<5!%<74`ISaOACjq|50@m(tPN!f~yv^eP?c zK|0cVbfnklNUzP29+@LOF-Lk}j`W-y={Y&lJ94CVRirMhm|Dlf5O<-gU=a-E@ssEyGL2xamPIqG!@6a&Bybq z(W|<>cJyI>#L_E+cQ~jALK=$8(qBcpm8dP8=_2pap3cG~a@*TI%W~% zqV$XEzohO-KBrhCQX}qx%^Ms=HjGXecSD8?~RiVj2-rMOvg#5V>)f?9HB{4?@sKU z^6Xq7ZP2OSo!Y5OeY=#`Wq-G_yW8W$+u@`aOYacLqpzHl*@w4J+x2Uoe#`^P^rd9F z16?B9y%A0Z%ShdQL7fcY9g@zM)XA_kBhrt^-vxg%rhUh><#;2t#mR(p6P72`ZSpE< z8zY^?^?%}<=41-%lzRDv71Jt68B^*$r7qLToNgj{PPdS{Os|q-FUi-p(#a)dUDifd z)aiqJ?FL8 zylo5O1@&6c77IAN4^G^4o-8_cQJohtFFOBA+Hy%7EJ?R4pJm6cIIk<}VZiCcP3eh2 zniDsfCk9&`sK=oj5J8qXwF#`8YLTd0J^-O)pK*Wx7iC*T|=4Gfr#euLti` z&)sRQblw-IwIgw=tL;=*+o_wrQ{8K)uKm+`dDpkm5Y3T1>bK&wVTz{Xw9)oP~HId(+*{K*w!Kc4t4D8j?=Em zIPGa7ZQ^F?v{(MU^6kalXM0~O$+J(MuI~e%fz&U_4F-n+gAr?L%0GJ= zUQ)M9j=wCwtMa<4&9ADbYw>i(z8Ps|)O|)?GmgC`pKIzgtNqhwOlI6>IdD_7B)UXnhdZP%WvKFZVNF?q@^N4q+Zq zhR?jSkx^2<&%Lveoj7w1osCwJ{iEtVD(&b-oVi&)^WAgKTFT zj(1yoW?=2iz}cDa=QFpqXPfG3KDo%9+4e9Yj>@QxARxO`4;5TA)Q*qGiJ92`1fK zqjlP#P1>Su+KJ@+q&fU^_~-D?;h)3reV*pDl`+=nOk;Yp`gTEM@6e*wRHljO5LE#P0k?>;3h;9tPM zfPVr10{#X33-}lCFW_Imzkq)Mzx$fBfZu0DTEOofC%MN-3-}lCFW_Imud^*J;9tPM zfPVr10{#X33-}lCFW_Im@BSz);9tPMfPVqM&!Mz{e*wRHr?iNlVJ9u(U&OzNe-Zy8 ze&1bb5&t6oMf{8S7x6FRU&QaeD=p$*#J`At5&t57&nJobCoSUFJCGLf`wmRZK#3VB zd2UII_!sdn;&<B^rA7RU_!sf(Bu$=$ z(jxvv{EPS(@h{?E#J`At5&t57&r4|${}O)BP02lD(u0wf@Gs$C!oP%n3I7s)9=gOs zmw4#X68P(zJx%zgSO8_?Pf6;WvPp zmhih@P3{rW682j_?Pi7<6p+VjDH#b zGX7=!%lMb^FXLavzl>jJby~*1jDH#bGX7=!%lQ49`Lv9G8Nd7Uw2Xfl|1$n%{LA>4 z@h{_F#=nez8UHf=W&F$dm+>#-U&glsUW#?lJ@75sY0k{+_Of`0}73VzS- zX$AiZ{uTTy_}w=q_l-$UT3W%sg5NVvTEV}9e+B;ve$V{L{YF~Bzk*+{Thi;6R`Bbr zPde+9=c1&;J~84YM!dv`ml*L9BVO{nlyu@Jo%o3nFX_lnjChF=FEQdJM!dv`ml*NV zDt<=1#E6#|@e(6m(&?Y{IV48Bw2Ge*FS++gtN0o55+hz>#7m5Li4iX`;w6Iwi4ia9 z8BC0Li4iX`;w47BWZ)n%;w60;i4iX`;w1wKi4iX`;w47BWKbb_#!Chkl7WT9h?f}g z5+h#H)0i0Xl7WWAh?hJECPuu(h?f}gl4ruih?f}g5+hz>#7m5Li4iX`;-xkGjChF= zFEQdJM!dv`ml*L9BVJ;}ON@Al5ic>~B?B3W5ic>~B}TktP$MzoB}Tl&h?f}g5+hz> z#7m5Li4iX`;w47B#E6&H@H65iM!dv`ml*MqzMI5|ml*L9BVN*Lni%ntF4N?mGBM&M zM!dv`ml*MqZq&qxmvp5jM!dv`m)v(I{XdBjFBxP>jChF=FBxn}23ry%URuY`h?f}g z5+hzR0FxN;5+hz>#7m5Li4iX`;w47BWN;=i;w47B!vK`$}rCHLn^7j)7EojkiG2ED|fml*UCgI>}t zof!0zKDET4ml*UCgI;3LOALC6K`(9KXV6Oquo8n_V$e$rdT9ecgI;3LOALBR_jY2? zOALC6K`$}rB?i63pqCi*lCJN>pqCi*5`$i1&`X{j(gyww{2TZ^M5Gy0`1{Q9kuVT{D^m$vXT{iQAZjDJagR??r9 zw(v6mrY-zi__y#IAWh7Gi5W04114s`q~9wUG)>Hai5W04119}o$q-30M3OvLCeM|L z889&eCT76I44Ag@GXp05Wr-Ou8A?gafQcC}=|4;QFOp}`#0;49VQ!o)zBcJS}u-@(sJm<$;uX2Qfwn3xIE4*nhdJNS3-@8D-J zObmvJ!7%OMXEIDohKb2AF&QQ%!^C8mm<*GC_rz$JcJS}u-@(6we+NJFVcNmZe3%SV zB?iRAfS4E%lOe05-!kpsXBtR5`1RkXot?-RFH#wmQw3F06;)FW)lwbRQv)?p6E#x{ zwNe|kQwMcY7j;t)^->@8(*O<95Dn7^jnWv6(*#Y@6iw45x=dH-D$UR}nx*S>gKp9s z&C>!c(h@Dx3a!!_t#Zv-IUM{A_)>3_ojMF2m2>m&@?8_~kPEY<{^6zdqw!hM(Op zm*FqN&-Rzg@U#BqGW`1Wav6T(E4d7R8U8Z-`kJ%8=FAqD_4{Skz|0z$4NGSBz-$~Q zm*Hm-%#04X48Oj?Y-}d$pU(QHb2)xPn7JH(IsS6|<@n3-m*X$TuMayj6K4I`xg38v z{&M{K7;`y({fxOBe>r~rjkz2@BT6pEUyk26P-Z~P<@n3-m*Zzf%;osY@t5N-$6t=0 z$1pP!=5qYy_{;Hg31%+A%n_K&@t5N-$6t=0uP&G4*SDUT`P?#3TV}e-JZ+h$Emz>@WXl!!EAVr&WlpwSf!~mBHg=VH*D}*u=2puU_$%=1 zQ_U6lIny#TT4vwMEN8g_KPy@05z7_$4GCvsUzxcivvOsAt<0sB^~YwOs?0By4H;*4 zsLZ&N*`YG)Q)Ye2%qW@RDKj=@hNsNMl)0EP_g1dJ&$5&mm9l=_%&3$Zm2w6C3j7uL z8J04`Qf64n3`?0|DKji(eZZM%DOcb(nJSx1l})C~#wN1KRJjtr$yC{7s%$7bn@p7} z@taJQO{U5wQ)QE>vdL80WU8z`Ih#zCEAg99l})J1mG~?1SK_b4Z-Q0U=bTNl$|hN5 zldN(jeiN;-alu@P-$biiiN6wmC4Q5xawYys{7gC7w)vdLk&3O`d)uEJl1pY0%5;jhA9g}(}a6@I3rT#cV`DOck+sVtjRmaFks z<2Si1SL3h7UyZ*SKZ8?dVaV0^*%&fcdS-aaTh{Hzn%#I|f= zTV|oiCbnhc7@4^$SL0`}%GLO*@mJ%o#$S!U8hWN!Ox+&UX4$;Pg;v65Va-{iSmgTDrU4So~pvT?p_GF`60UxS~~ zE1OuC**&t!b(!HS*Wj#>_038P2kC$6SlQ7C-Y@ zW<$wl5#(C@wfK!m=34x<_-pYSx6fu5WU~u$E&f{kwfJlCGp=Rhm$?>yEq>#exfVY& zTdu`#Of%Qwuf<=BzZSn43b_`)SqhnXG4lgt#<$E9ka+@fE&f{kEHSwje=Yu6{I&RN z@f)+sd;^(pAlKqIewAzSGtOl`f?SKg7Jn^%o`PJ5-}HuDhrbTLX%4v#KZ9N7H^_DP zO?k+5_*rjq9e&2UT!+67KR-gQ!(WG=EhpFEuft!5zYc#Letv~qho5I5*WqW~$#wYa z@Ymt5!_Ui*>+sj%uft!5zYc#L{yO}1`0Mc3;WzCf8w<|Hf^!{y(=M_x;arE`^owlz zMK)%a83VI%;%uBa*Ws_jZyYbz;jhDAho5&Mo1T&D@z>)wO(UD8k?Zl-<2PL+GZ*H1 z{KlGdJ^p(9_4w=Y*W)+-oa^z|+#p)ugA~hkxd)Prj2A?k6e%6xOc9{Z|plWKIVG-_4w=Y z*W<6pUyr{Ye?5NV9Og0@RH{fr;-+;dXe*^vo{0;b-JaYs72K){989p<^XKujH z7n2+Cvxnsd{EhgTgL5PPM*PNhGh=9G49)CgxedZ^Yk-zY%{U{zm+b_?et@BYu8|+=!o{H8E~&e>47O{JdPb z8GkeWX8a7uxfy>m{$~8m_?z)Jpv&`l;{B8K# z@VDV_!{3I#4SyT{HvGJWxeb3C{xXB;KcjAL$KQ^>9e+FicKq%5+wr&KZ^z$`za2l{WNydLLY&+2 zx8rZe&&Z$K@wek|$IsNC+wr&KZ^z$`za4)&{&xKB_}lTf<8Q~`j=vp$JN|b3?fBdA zx8rZe-;Tc>e>;8?b8F8p2iyYP46@50}OzYBjC{x1Ao_`C3T;qSuVg})1b7yd5%UHDBz$*l3Y z3x5}W{@>h%zYBjCev_?o7yd5%UHDDB%3b)o@ORF8p2iyYTbveVa{N4Dw@pt3z#@~&f$1r!}@5XPMLGH%ijo&1n z+>O5*zsWwC4>5P+Hw__oF!QX?w2R}b} zHa#Nu;P1iTgTDuV5B?tfJ@|X@_u%ir--Evge-Hj1{5|-4@b}>F!QX?w2Y(O#9{fG{ zd+_(*@4?@LzXyLW{$BjO_l@$<0fUi`ej z*?f-Ni@z6tFaBQqz4&|a_u}ux&lj3`-g7U0zV~eYNAAVni@z6tFaBQqz4&|a_u}ux z-;2K&e=q)C{Jf>P7k@AQUi`iId-3<;H*qod;_t=Zi@z6tFaBQqz4&|a_u}VM&VBg% z@blm2KKy<7`|$VS@5A4R-^7&Mho6@<_u=Pf&3*X$@bfrllgM%({yzMD`1|np;qSxW zhrbVhAO1f4{Lr}%e;@un{C)WQ@b}^G!{3L$4}TwizUgdoTIQY3efasOb07Xb{C)WQ z@b}^G!{3L$55EaCxevd21i24C&u#9*--o{sKmTnuzaaPF@5A4ZzaM`;{(k)Z`1|qm z=;nU>{Mxx6e?R_y{Qda*@%Q8J$KQ|NWS!iPzaM`;{(k)Z`1|qq=#hJG__v7!!-;ci^zlp8c{E9q)e*pgg{sH_0_y_Qtbd(4158xlbKY)J# z{{a30`~&z0@DJc0z(0V00RI5~0sI5_2k;NzAHY9=e*pgg{sH_0_y_P0;2*%x_ni5j z^8o$<`~&#S_sD$Ec>wC8#od@unFqH@J z58xlbKY)J#{{a30{3cUnb4l_b{z3eM_y_S1;vd9sqE$B0Di7ix#6O6C5I^sJ9>hP0 ze-Qs5{z3eM_y_TuK$%U>%7gd^@ekrRQ7aGPAH;9&N;Y>T58^j@D-Yu5^Uj0#2k{T$ z=T*+;2joHggZKyW58@xhKZt)2zsZ?-5I=u-9>hP0e-Qs5{z3eM_y_S1;vd96h<_0O zApSx8L->dA58)reKZJh>{}BEm{5%fNA^b!5hwz(dmWS{U;pa)uL-FMA%sKZJh>{}BEm{6qM8-188A^Ca>R z{vrHB`1!c=5dIdA58)reKZJh>{}6t2EAkM2li~ck6T&})e+d6DeiPL5F#ciu z!}v{@%jRO_Vf@4Rhw%^NH@_$k;~&O9jDHyaF#ciu!}y2s591%kKaAg`yF84482>Q- zVf@4Rhw%^NAI3k7e;EHT{$c#X_=oWi<2RQn591%kKa77E|1kbx{KNQ%@eku4#y^aI z82>Q-Vf@4Rhw%^NAI5K9Ngl>OjDHyaF#ciuBlt(~kKiA{KZ1V*{|NpO{3G~B@Q>g( z&mfQBAHi=ja2~-wf`0`62>ucLBlt(~kKiA{KZ1V*{|J6_6!HlE5&R?gNAR1okj;0= z=4<5<{3G~B@Q>gh!9Rk31pf&B5&R?gNAQo}AHhF@e+2&se)Ar(3Cei{{|NpO{3G~B z@Q>gh!9Rk31pf&B5&Wb0&6CKZ_($=N;x}g^kK!N2KZ<`8|0sU*DDo(N6QJ`b{!#p+ z_($=N;vdC7ihmUUD1H+yvw0VJ6#ppxQT(I$NAZv1H@_#3;vdC7ihmUUDE?9WqxeVh zkK#9PBah-A#XpLF6#poGKdO*N@tcgDO~%fn_($=N;vdC7ihmUUDE?9W=6>W+{G<3s z@sHv+zbKF4AHzR}e+>T^{xSSx_{Z>%;UB|4hTmMIJci%Ax;%z|4F4E@b4T+S{xSSx z_{Z>%;UB|4hTkOdJcfS^{}_IAQ1Tdl^P2J){xSUKH{~(>WBAALkKrG~KZbt{{}}!; z{A2jX@Q>ji!#{?94F4GZG5llr$MBEgAHzR}e+>T^{xSSx_{Z>%;UC9u&TAgWKaPJK zzxl6u9RE1}as1=>$MKKjH;*un<2OGxkK-T5KaPJK|2Y0}{Nwn?@sHyl$3KpL9RE1} zas1=>$MKKjAICqAe;off{&D={_{Z^&<2O$ykK-T5KaPJK|2Y0}{Nwn?@sHyl$8X|& z9>+h9e;off{&D={_{Z^&;~&RAj(;5gIQ|Lz6Zp+5%MN|@K4~Mz;C`$p1?nWe*(X`M|lFjIc(V+ zq&$It0{;a53H%fIC-6_;pTIwXe**sm{t5i%EM-3|kSFl_ae+L6e**sm{t5hkY#^H( zmnZO>AD7Mf%;q@d3H%fIC-6_=pTzG+3GyWVN&J)e{XjvU#6O9D68|LrN&J)eC-G0> zpTuv@U7o~0iGLEm`O?{s8stg*llUj`PvZ9@2YC{|A3DgB_$TpC;-ADniQkVOQK2|nYHBaK7#6O9D68|LrN&J)eC-G0>pTs|je-i&B{z?3k_|0?8 zllaYb&6D^i@lWEP!f(!CHpe?p;h(}kg?|eF6#gmvQ~0OwPvM`!KZSn^{}g_6WwT#r z$W!>I@cWg9JcZx<**t}R3jY-TDg0CTr|_Fso2T$k;h(}kh2Jkb^A!Fm z{8RX+@K52N!as$73jY-TDg0CTr|?hV_sbA@3jY-TDg0CTr|?hVpTa+de+vH;e)A)< zIlXxrzj?iR8viu@Y5aa(Et@-;r}0nYpT<9p-(1Q(jei=yIl$Q*;5?0g8viu@Y5ddp zr}3LVm#6Vh|36jTOKxS2x@Kjv7Zhm&P0SQo@TC9@@$=ifVgw;?Hk5DnrRujD51 z?1Y4*;2B;aYel@F2zz ze(l};vHy?#f9(HbzZUQQ*sndlKlcBz|BwBD?Eho`AN&8<|HuA6_W!Z}kNtn_|6@Oq z`Tp3iS-wB^zu5m`zrOEY?0>QU#r_xjU+jOe|HXcN^S#*rV*iW%FZRFK|6>1({rb{- zvH!*X7yDoAf3g3?{uldS?0>QU#r_xjU+jOe|Hb|n`(NzWr{0VGFZRFK|6>1({V(>v z*#Bbxi~TS5zu5m`|BL;)#Cx&-#r_xjU+jOe|Hb|n`(NyTvH!*X7yDoA*D>CU{d(AY zwV!KnulB#%|7!oM{jc`F+W%_*tNpL`zuNz5|EvA4_P^TyYX7VKT!eeI|JD9i`(N#U zwg1)rSNmV>*M{G#{jc`F+W%_*tNpL`zuNz5|EvA4_LEKS)&5ueU+sUj|JD9i`=Lqq zYX7VKulB#%|7!oM{jc`F+W%_*tNpL`zuNz5KXm(E?SHlZ)qb7&z1ja}|C{}9_P^Qx zX8)W0Z}z|0|7QQ2{crZa+5cw$oBeP0zuEt0|C{}9_P^QxX8)W0Z}z|0|7QQ2{crZa z+5cw$oBeP0zuEt0|C{}9_P^OrbGbMB-|T<0UvqkI_P^QxX8)W0Z}z|0|7QQ2{crYb z?eES0H~Zi0f3yG1e*OKu+5cw$oBeP0zuEt8|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_ z+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~ z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g z{qOd_+y8F=yZ!I>zuW&||A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F z|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIP z|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIPuV+8b!hP8PVgHByANFh9 z@5BC2`?c=(Y5%AFER*}R|I_|Y`)NY=X+QM)KJ5p8->3cX@B6g>)BaEUHPH8I|EK+* z_J7*{Y5%AFK(qU_|I_|Y`#&x%U z{xAE#?EkX=%lzwBqv+?W0QnftQ;%lzwH0A|I2>1!F}8RZU49Z-}Y0Y?%V!v z`@ik~wx54+-}Zmo|84)b{oIcGw*TAyZ~MRP_kR-HxBcJtlcw(5{%`xg?fZANzmo|FK{HfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{@;J0>&<@s|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs< zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a zpZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmv zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>! z|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^ z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y z{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ< z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs< zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a zpZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmv zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>! z|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^ z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y z{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ< z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnx*Y@lG z)BmUcPye6(zkhAN{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(zkg%D{y+VH`v3I*>HqsT_Ur%C|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|9i0i!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT z!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?AQON|4;v){y+VH`v3I*>HmAM z|H1wT`ycGr|EK@&!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT!TtyP z_5bPr)BmUcPye6(KmC9D{~qmswExlmNBi~v>HmAQU;p2u{g3uP+W%<(qy3NeKidCj z|D*kn_CMPHX#b=AkM=*>|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*> z|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*>|7ib{{ZIBk+5cq!ll@Qj zKiU6e|C9at|MdUq|9i6k$^Iw%pX}HFr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGqd$#}C{%8B2?SHoa+5Tty zpY4CP|Ji>1fBOHP?SHoa+5Tty_5bPr)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIa z#r}V>U;m%}KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq`^)~n?AQON|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|L^VJe_6r+gaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu1JJwu@Akjj|8749APhk7 z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^VY0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da^kM&p{U7#!*pC4S0}uuv z3_u_Df7t(F|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy9=Sf7<_P|EK+*_J7*{ zY5%AFpZ0&+|7riH{h#)K+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+|7riH{h#)K z+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+j{yh+(5L;M_J7*{X+H)a3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE)om;GP%f7$a0Q6=5m;GP%f7$a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaPpOf2ysc0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaM~?`)L5t0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0jb`1?r%hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR*uS=)1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0Q)!g(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|*iQo(4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|+D`)*4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e7Y zll@QjKiU6eKMi0sfYAU(1K5-OPxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMG zWdD=>Pxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMGWdD=>Pxe3A|78D@{ZIBk z+5cq!ll@QjKiU6e|C9Ys_CMMGWdF1M&-Opt|7`!W{m=G4+y89;v;EKZKimIo|Fiwi z_CMSIZ2ze+fM@+4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y-~ZBuLWlqUABPVI*LfU%9Grb|cyjPM-r;}$$Kn5Q@TJ|s&;1Yo#eeJV@Bet` zzx)5as9@IU>>5B}%>_`$#P;NSkQKJ*_?`G3FT!9Skq|9$X}XZpAQ z+kg0wfB(*Z_>lkbz=!;kt6RVS94i64`o!@^E2>kx#(*5sW&V&B`<%HAkU(V7v_`Uq)e22q--R6=(C=UW&Y$vM?$!PM%S}$df4OVr_b<18{Qk|k>fgUP3HkdsXJvl>=4Ha~ z-~7A#{TmM8_it{(|NWa=!hiqf-VO)9_}^Vw|NXnmln(!kgNuR=|9H>o_20icC-D1s zU-WT-*6$zROZ@)v;_>$nQ^LV-?a!y@`}2?A+8=jl|Ne11?e8D=6aD^i z1Ih1;E4aTePV@Y}_`3W1A~*ki5##>8xCQ<9#l2m>FK#&cef3Ju!5`SG&q2Sh`ikFI zciR5Gxor6R=9', '.']) + + def test_sequence_builders(self): + tokenizer = AlbertTokenizer(SAMPLE_VOCAB) + + text = tokenizer.encode("sequence builders") + text_2 = tokenizer.encode("multi-sequence build") + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_2 + [tokenizer.sep_token_id] + + +if __name__ == '__main__': + unittest.main() diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index f2e37222f6..0785e55ad2 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -8,6 +8,7 @@ from shutil import copyfile logger = logging.getLogger(__name__) +VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'} SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): @@ -16,12 +17,12 @@ class AlbertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ - # vocab_files_names = VOCAB_FILES_NAMES + vocab_files_names = VOCAB_FILES_NAMES # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, - do_lower_case=False, remove_space=True, keep_accents=False, + do_lower_case=True, remove_space=True, keep_accents=False, bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, @@ -142,15 +143,15 @@ class AlbertTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An ALBERT sequence has the following format: + single sequence: [CLS] X [SEP] + pair of sequences: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] if token_ids_1 is None: - return token_ids_0 + sep + cls - return token_ids_0 + sep + token_ids_1 + sep + cls + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): """ @@ -175,25 +176,24 @@ class AlbertTokenizer(PreTrainedTokenizer): return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) if token_ids_1 is not None: - return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1] - return ([0] * len(token_ids_0)) + [1, 1] + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: - 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 - | first sequence | second sequence | CLS segment ID + An ALBERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence if token_ids_1 is None, only returns the first portion of the mask (0's). """ sep = [self.sep_token_id] cls = [self.cls_token_id] - cls_segment_id = [2] if token_ids_1 is None: - return len(token_ids_0 + sep + cls) * [0] - return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index a4f1a6e3ba..c01fbbbeeb 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -185,9 +185,9 @@ class XLNetTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An XLNet sequence has the following format: + single sequence: X + pair of sequences: A B """ sep = [self.sep_token_id] cls = [self.cls_token_id] @@ -224,7 +224,7 @@ class XLNetTokenizer(PreTrainedTokenizer): def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: + An XLNet sequence pair mask has the following format: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 | first sequence | second sequence | CLS segment ID From 1e5b31c3881cb1313216ce0f3cffae89b0845d4f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:25:32 +0000 Subject: [PATCH 085/143] Several fixes and improvements --- transformers/modeling_albert.py | 36 +++++++++--------- .../{30k-clean.model => spiece.model} | Bin transformers/tokenization_albert.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) rename transformers/tests/fixtures/{30k-clean.model => spiece.model} (100%) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index ad8b979cef..371a2e535c 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -7,6 +7,7 @@ import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from transformers.modeling_utils import PreTrainedModel from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -37,18 +38,17 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): print(name) for name, array in zip(names, arrays): - print(name) - og = name + original_name = name name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") - name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("LayerNorm_1", "attention/LayerNorm") + name = name.replace("cls/predictions", "predictions") + name = name.replace("transform/", "") + name = name.replace("LayerNorm_1", "full_layer_layer_norm") + name = name.replace("LayerNorm", "attention/LayerNorm") name = name.replace("inner_group_", "albert_layers/") name = name.replace("group_", "albert_layer_groups/") name = name.split('/') - - print(name) pointer = model for m_name in name: if re.fullmatch(r'[A-Za-z]+_\d+', m_name): @@ -78,13 +78,12 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): pointer = getattr(pointer, 'weight') elif m_name == 'kernel': array = np.transpose(array) - print("transposed") try: assert pointer.shape == array.shape except AssertionError as e: e.args += (pointer.shape, array.shape) raise - print("Initialize PyTorch weight {} from {}".format(name, og)) + print("Initialize PyTorch weight {} from {}".format(name, original_name)) pointer.data = torch.from_numpy(array) return model @@ -177,9 +176,9 @@ class AlbertAttention(BertSelfAttention): b = self.dense.bias projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b - projected_context_layer = self.dropout(projected_context_layer) - layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) - return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + projected_context_layer_dropout = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) + return layernormed_context_layer class AlbertLayer(nn.Module): @@ -187,17 +186,17 @@ class AlbertLayer(nn.Module): super(AlbertLayer, self).__init__() self.config = config - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.full_layer_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask)[0] + attention_output = self.attention(hidden_states, attention_mask) ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) return hidden_states @@ -352,16 +351,17 @@ class AlbertModel(BertModel): encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) + sequence_output = encoder_outputs[0] - + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) - outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(nn.Module): +class AlbertForMaskedLM(PreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -384,7 +384,7 @@ class AlbertForMaskedLM(nn.Module): """ def __init__(self, config): - super(AlbertForMaskedLM, self).__init__() + super(AlbertForMaskedLM, self).__init__(config) self.config = config self.bert = AlbertModel(config) diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/spiece.model similarity index 100% rename from transformers/tests/fixtures/30k-clean.model rename to transformers/tests/fixtures/spiece.model diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 0785e55ad2..7b16bb573f 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -8,7 +8,7 @@ from shutil import copyfile logger = logging.getLogger(__name__) -VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'} +VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): From 5680a1106302b1ebeb960de0700d6379c0aeef5c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:42:49 +0000 Subject: [PATCH 086/143] Activation function managed from the config file --- transformers/configuration_albert.py | 2 +- transformers/modeling_albert.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index c86c9565cb..15437dbbea 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -16,7 +16,7 @@ class AlbertConfig(PretrainedConfig): intermediate_size=16384, inner_group_num=1, down_scale_factor=1, - hidden_act="gelu", + hidden_act="gelu_new", hidden_dropout_prob=0, attention_probs_dropout_prob=0, max_position_embeddings=512, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 371a2e535c..7e9f7f1c46 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from transformers.modeling_utils import PreTrainedModel from .file_utils import add_start_docstrings @@ -190,11 +190,12 @@ class AlbertLayer(nn.Module): self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + self.activation = ACT2FN[config.hidden_act] def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_output = self.attention(hidden_states, attention_mask) ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) + ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) @@ -392,6 +393,7 @@ class AlbertForMaskedLM(PreTrainedModel): self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] def tie_weights(self): """ Make sure we are sharing the input and output embeddings. @@ -405,7 +407,7 @@ class AlbertForMaskedLM(PreTrainedModel): outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] hidden_states = self.dense(sequence_outputs) - hidden_states = gelu_new(hidden_states) + hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) prediction_scores = self.word_embeddings(hidden_states) From ce9eade29c75fa676ac528d1fe21d9f4ac3c5622 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:50:44 +0000 Subject: [PATCH 087/143] Initializer range using BertPreTrainedModel --- transformers/modeling_albert.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 7e9f7f1c46..b45208b696 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -6,8 +6,7 @@ import torch import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN -from transformers.modeling_utils import PreTrainedModel +from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -362,7 +361,7 @@ class AlbertModel(BertModel): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(PreTrainedModel): +class AlbertForMaskedLM(BertPreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. From 25a31953e8820eb5c88d8ad35ee547efccfe577c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 21:18:06 +0000 Subject: [PATCH 088/143] Output Attentions + output hidden states --- transformers/modeling_albert.py | 58 ++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b45208b696..52cab2ea69 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -105,6 +105,7 @@ class AlbertAttention(BertSelfAttention): def __init__(self, config): super(AlbertAttention, self).__init__(config) + self.output_attentions = config.output_attentions self.num_attention_heads = config.num_attention_heads self.hidden_size = config.hidden_size self.attention_head_size = config.hidden_size // config.num_attention_heads @@ -177,7 +178,7 @@ class AlbertAttention(BertSelfAttention): projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b projected_context_layer_dropout = self.dropout(projected_context_layer) layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) - return layernormed_context_layer + return (layernormed_context_layer, attention_probs) if self.output_attentions else (layernormed_context_layer,) class AlbertLayer(nn.Module): @@ -193,25 +194,45 @@ class AlbertLayer(nn.Module): def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_output = self.attention(hidden_states, attention_mask) - ffn_output = self.ffn(attention_output) + ffn_output = self.ffn(attention_output[0]) ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output[0]) - return hidden_states + return (hidden_states,) + attention_output[1:] # add attentions if we output them class AlbertLayerGroup(nn.Module): def __init__(self, config): super(AlbertLayerGroup, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for albert_layer in self.albert_layers: - hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + layer_hidden_states = () + layer_attentions = () - return hidden_states + for albert_layer in self.albert_layers: + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + layer_output = albert_layer(hidden_states, attention_mask, head_mask) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + return outputs # last-layer hidden state, (layer hidden states), (layer attentions) class AlbertTransformer(nn.Module): @@ -227,11 +248,30 @@ class AlbertTransformer(nn.Module): def forward(self, hidden_states, attention_mask=None, head_mask=None): hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + for layer_idx in range(self.config.num_hidden_layers): group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) - return (hidden_states,) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in From 870320a24e28f187d0dfd10b82c4d60d5269374d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 22:30:21 +0000 Subject: [PATCH 089/143] Early tests --- transformers/modeling_albert.py | 59 +++---- transformers/tests/modeling_albert_test.py | 191 +++++++++++++++++++++ 2 files changed, 219 insertions(+), 31 deletions(-) create mode 100644 transformers/tests/modeling_albert_test.py diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 52cab2ea69..f906352311 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -11,6 +11,15 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) + +ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", +} + + def load_tf_weights_in_albert(model, config, tf_checkpoint_path): """ Load tf checkpoints in a pytorch model.""" try: @@ -39,6 +48,7 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): for name, array in zip(names, arrays): original_name = name name = name.replace("ffn_1", "ffn") + name = name.replace("/bert/", "/albert/") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions", "predictions") @@ -114,29 +124,6 @@ class AlbertAttention(BertSelfAttention): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.pruned_heads = set() - def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.num_attention_heads, self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - - # Prune linear layers - self.query = prune_linear_layer(self.query, index) - self.key = prune_linear_layer(self.key, index) - self.value = prune_linear_layer(self.value, index) - self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) - - # Update hyper params and store pruned heads - self.num_attention_heads = self.num_attention_heads - len(heads) - self.all_head_size = self.attention_head_size * self.num_attention_heads - self.pruned_heads = self.pruned_heads.union(heads) - def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) mixed_key_layer = self.key(input_ids) @@ -225,7 +212,7 @@ class AlbertLayerGroup(nn.Module): layer_attentions = layer_attentions + (layer_output[1],) if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) + layer_hidden_states = layer_hidden_states + (hidden_states,) outputs = (hidden_states,) if self.output_hidden_states: @@ -367,6 +354,8 @@ class AlbertModel(BertModel): self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() + self.init_weights() + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: @@ -422,33 +411,41 @@ class AlbertForMaskedLM(BertPreTrainedModel): list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) self.config = config - self.bert = AlbertModel(config) + self.albert = AlbertModel(config) self.LayerNorm = nn.LayerNorm(config.embedding_size) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) - self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) self.activation = ACT2FN[config.hidden_act] + self.init_weights() + self.tie_weights() + def tie_weights(self): """ Make sure we are sharing the input and output embeddings. Export to TorchScript can't handle parameter sharing so we are cloning them instead. """ - self._tie_or_clone_weights(self.classifier.word_embeddings, - self.transformer.embeddings.word_embeddings) + self._tie_or_clone_weights(self.decoder, + self.albert.embeddings.word_embeddings) def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): - outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] hidden_states = self.dense(sequence_outputs) hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) - prediction_scores = self.word_embeddings(hidden_states) + prediction_scores = self.decoder(hidden_states) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here if masked_lm_labels is not None: diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py new file mode 100644 index 0000000000..46a2eeb729 --- /dev/null +++ b/transformers/tests/modeling_albert_test.py @@ -0,0 +1,191 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +from transformers import is_torch_available + +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +if is_torch_available(): + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM) + from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP +else: + pytestmark = pytest.mark.skip("Require Torch") + + +class AlbertModelTest(CommonTestCases.CommonModelTester): + + all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () + test_pruning = False + test_head_masking = False + + class AlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertModel(config=config) + model.eval() + sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output, + "pooled_output": pooled_output, + } + self.parent.assertListEqual( + list(result["sequence_output"].size()), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size]) + + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForMaskedLM(config=config) + model.eval() + loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) + result = { + "loss": loss, + "prediction_scores": prediction_scores, + } + self.parent.assertListEqual( + list(result["prediction_scores"].size()), + [self.batch_size, self.seq_length, self.vocab_size]) + self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = AlbertModelTest.AlbertModelTester(self) + self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = AlbertModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From c14a22272f3fc17bb2eaeca62986c31a7d26bc85 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 14:04:10 +0000 Subject: [PATCH 090/143] ALBERT passes all tests --- transformers/configuration_albert.py | 4 +--- transformers/modeling_albert.py | 9 +++------ transformers/tests/tokenization_albert_test.py | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index 15437dbbea..04f9fa8d60 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -7,7 +7,7 @@ class AlbertConfig(PretrainedConfig): """ def __init__(self, - vocab_size_or_config_json_file, + vocab_size_or_config_json_file=30000, embedding_size=128, hidden_size=4096, num_hidden_layers=12, @@ -15,7 +15,6 @@ class AlbertConfig(PretrainedConfig): num_attention_heads=64, intermediate_size=16384, inner_group_num=1, - down_scale_factor=1, hidden_act="gelu_new", hidden_dropout_prob=0, attention_probs_dropout_prob=0, @@ -61,7 +60,6 @@ class AlbertConfig(PretrainedConfig): self.num_hidden_groups = num_hidden_groups self.num_attention_heads = num_attention_heads self.inner_group_num = inner_group_num - self.down_scale_factor = down_scale_factor self.hidden_act = hidden_act self.intermediate_size = intermediate_size self.hidden_dropout_prob = hidden_dropout_prob diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index f906352311..9bb38dead9 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -202,17 +202,14 @@ class AlbertLayerGroup(nn.Module): layer_attentions = () for albert_layer in self.albert_layers: - if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) - layer_output = albert_layer(hidden_states, attention_mask, head_mask) hidden_states = layer_output[0] if self.output_attentions: layer_attentions = layer_attentions + (layer_output[1],) - if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) outputs = (hidden_states,) if self.output_hidden_states: @@ -247,7 +244,7 @@ class AlbertTransformer(nn.Module): hidden_states = layer_group_output[0] if self.output_attentions: - all_attentions = all_attentions + layer_group_output[1] + all_attentions = all_attentions + layer_group_output[-1] if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) diff --git a/transformers/tests/tokenization_albert_test.py b/transformers/tests/tokenization_albert_test.py index dd63f6756b..59eb3bceb0 100644 --- a/transformers/tests/tokenization_albert_test.py +++ b/transformers/tests/tokenization_albert_test.py @@ -22,7 +22,7 @@ from transformers.tokenization_albert import (AlbertTokenizer, SPIECE_UNDERLINE) from .tokenization_tests_commons import CommonTestCases SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'fixtures/30k-clean.model') + 'fixtures/spiece.model') class AlbertTokenizationTest(CommonTestCases.CommonTokenizerTester): From b21402fc86257feca05ab050d061e15441a49929 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 14:19:31 +0000 Subject: [PATCH 091/143] Python 2 tests + licence --- transformers/modeling_albert.py | 16 ++++++++++++++++ transformers/tokenization_albert.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 9bb38dead9..b6d1291725 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -1,4 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch ALBERT model. """ + import os import math import logging diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 7b16bb573f..7cba99b9e4 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -1,4 +1,21 @@ - +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization classes for ALBERT model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + from .tokenization_utils import PreTrainedTokenizer import logging import unicodedata From c4403006b8301a24bfeec99c39ce8d5d47df570f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 15:30:11 +0000 Subject: [PATCH 092/143] External MLM head --- transformers/configuration_albert.py | 17 ++++++++++++++ transformers/modeling_albert.py | 35 +++++++++++++++++++--------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index 04f9fa8d60..b72bbb971e 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -1,3 +1,20 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ALBERT model configuration """ + from .configuration_utils import PretrainedConfig class AlbertConfig(PretrainedConfig): diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b6d1291725..487455e561 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -401,6 +401,26 @@ class AlbertModel(BertModel): outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs +class AlbertMLMHead(nn.Module): + def __init__(self, config): + super(AlbertMLMHead, self).__init__() + + self.LayerNorm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.decoder(hidden_states) + + prediction_scores = hidden_states + self.bias + + return prediction_scores + @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(BertPreTrainedModel): @@ -433,13 +453,8 @@ class AlbertForMaskedLM(BertPreTrainedModel): def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) - self.config = config self.albert = AlbertModel(config) - self.LayerNorm = nn.LayerNorm(config.embedding_size) - self.bias = nn.Parameter(torch.zeros(config.vocab_size)) - self.dense = nn.Linear(config.hidden_size, config.embedding_size) - self.decoder = nn.Linear(config.embedding_size, config.vocab_size) - self.activation = ACT2FN[config.hidden_act] + self.predictions = AlbertMLMHead(config) self.init_weights() self.tie_weights() @@ -448,17 +463,15 @@ class AlbertForMaskedLM(BertPreTrainedModel): """ Make sure we are sharing the input and output embeddings. Export to TorchScript can't handle parameter sharing so we are cloning them instead. """ - self._tie_or_clone_weights(self.decoder, + self._tie_or_clone_weights(self.predictions.decoder, self.albert.embeddings.word_embeddings) def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] - hidden_states = self.dense(sequence_outputs) - hidden_states = self.activation(hidden_states) - hidden_states = self.LayerNorm(hidden_states) - prediction_scores = self.decoder(hidden_states) + + prediction_scores = self.predictions(sequence_outputs) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here if masked_lm_labels is not None: From 4f3a54bfc8fa8749f6d5b29f110148738a646fcd Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 16:37:34 +0000 Subject: [PATCH 093/143] ALBERT can load pre-trained models. Doesn't inherit from BERT anymore. --- transformers/__init__.py | 2 +- transformers/configuration_albert.py | 9 ++++++ transformers/modeling_albert.py | 44 +++++++++++++++++++++++----- transformers/tokenization_albert.py | 25 +++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 152d520e7b..bdfb1a0922 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index b72bbb971e..c35426768f 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -17,12 +17,21 @@ from .configuration_utils import PretrainedConfig +ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", +} + class AlbertConfig(PretrainedConfig): """Configuration for `AlbertModel`. The default settings match the configuration of model `albert_xxlarge`. """ + pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP + def __init__(self, vocab_size_or_config_json_file=30000, embedding_size=128, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 487455e561..4da10ed1cb 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -21,6 +21,7 @@ import logging import torch import torch.nn as nn from torch.nn import CrossEntropyLoss +from transformers.modeling_utils import PreTrainedModel from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings @@ -274,6 +275,29 @@ class AlbertTransformer(nn.Module): return outputs # last-layer hidden state, (all hidden states), (all attentions) + +class AlbertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + def _init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if isinstance(module, (nn.Linear)) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents @@ -338,7 +362,7 @@ ALBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertModel(BertModel): +class AlbertModel(AlbertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` @@ -358,6 +382,12 @@ class AlbertModel(BertModel): list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -369,6 +399,11 @@ class AlbertModel(BertModel): self.init_weights() + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: @@ -423,7 +458,7 @@ class AlbertMLMHead(nn.Module): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(BertPreTrainedModel): +class AlbertForMaskedLM(AlbertPreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -445,11 +480,6 @@ class AlbertForMaskedLM(BertPreTrainedModel): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - config_class = AlbertConfig - pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP - load_tf_weights = load_tf_weights_in_albert - base_model_prefix = "albert" - def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 7cba99b9e4..acf67c1154 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -15,7 +15,7 @@ """ Tokenization classes for ALBERT model.""" from __future__ import (absolute_import, division, print_function, unicode_literals) - + from .tokenization_utils import PreTrainedTokenizer import logging import unicodedata @@ -24,8 +24,25 @@ import os from shutil import copyfile logger = logging.getLogger(__name__) - VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'albert-base': 512, + 'albert-large': 512, + 'albert-xlarge': 512, + 'albert-xxlarge': 512, +} + SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): @@ -35,8 +52,8 @@ class AlbertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ vocab_files_names = VOCAB_FILES_NAMES - # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, do_lower_case=True, remove_space=True, keep_accents=False, From c9875455929a63f12b81e2dcc8fe30f72137c06b Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 18:48:02 +0000 Subject: [PATCH 094/143] Converting script --- transformers/__init__.py | 2 +- ...lbert_original_tf_checkpoint_to_pytorch.py | 36 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index bdfb1a0922..db98d5fd44 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py index 04877d41b9..5bbaab8c21 100644 --- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -1,18 +1,39 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert ALBERT checkpoint.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import argparse +import torch -from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert - +from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_albert +import logging +logging.basicConfig(level=logging.INFO) def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # Initialise PyTorch model - config = BertConfig.from_json_file(bert_config_file) + config = AlbertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) - model = BertForPreTraining(config) + model = AlbertForMaskedLM(config) # Load weights from tf checkpoint - load_tf_weights_in_bert(model, config, tf_checkpoint_path) + load_tf_weights_in_albert(model, config, tf_checkpoint_path) # Save pytorch-model print("Save PyTorch model to {}".format(pytorch_dump_path)) @@ -31,7 +52,7 @@ if __name__ == "__main__": default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained BERT model. \n" + help = "The config json file corresponding to the pre-trained ALBERT model. \n" "This specifies the model architecture.") parser.add_argument("--pytorch_dump_path", default = None, @@ -40,5 +61,6 @@ if __name__ == "__main__": help = "Path to the output PyTorch model.") args = parser.parse_args() convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, - args.bert_config_file, + args.albert_config_file, args.pytorch_dump_path) + \ No newline at end of file From 0d07a23c04c9837234cda402b32246d7581e5bc4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 15:07:01 +0000 Subject: [PATCH 095/143] LAMB implementation --- transformers/optimization.py | 93 ++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/transformers/optimization.py b/transformers/optimization.py index 99e6cc75e4..90f3dbca9b 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -167,3 +167,96 @@ class AdamW(Optimizer): p.data.add_(-group['lr'] * group['weight_decay'], p.data) return loss + + + +class Lamb(Optimizer): + """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training). + + Adapted from the huggingface/transformers ADAM optimizer + Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py + Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py + + + Parameters: + lr (float): learning rate. Default 1e-3. + betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) + eps (float): Adams epsilon. Default: 1e-6 + weight_decay (float): Weight decay. Default: 0.0 + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): + if lr < 0.0: + raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, + correct_bias=correct_bias) + super(Lamb, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('LAMB does not support sparse gradients.') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + # Decay the first and second moment running average coefficient + # In-place operations to update the averages at the same time + exp_avg.mul_(beta1).add_(1.0 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) + denom = exp_avg_sq.sqrt().add_(group['eps']) + + + # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py + step_size = group['lr'] + weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) + + adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) + if group['weight_decay'] != 0: + adam_step.add_(group['weight_decay'], p.data) + + adam_norm = adam_step.pow(2).sum().sqrt() + if weight_norm == 0 or adam_norm == 0: + trust_ratio = 1 + else: + trust_ratio = weight_norm / adam_norm + + + state['weight_norm'] = weight_norm + state['adam_norm'] = adam_norm + state['trust_ratio'] = trust_ratio + + p.data.add_(-step_size * trust_ratio, adam_step) + return loss From 6637a77f807615ef2427c0390a015f4eb4814fb4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 15:17:31 +0000 Subject: [PATCH 096/143] AlbertForSequenceClassification --- transformers/__init__.py | 3 +- transformers/modeling_albert.py | 77 ++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index db98d5fd44..51995942ce 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,8 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 4da10ed1cb..bba6767079 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -20,10 +20,10 @@ import math import logging import torch import torch.nn as nn -from torch.nn import CrossEntropyLoss +from torch.nn import CrossEntropyLoss, MSELoss from transformers.modeling_utils import PreTrainedModel from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN +from transformers.modeling_bert import BertEmbeddings, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -510,3 +510,76 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): outputs = (masked_lm_loss,) + outputs return outputs + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForSequenceClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base') + model = AlbertForSequenceClassification.from_pretrained('albert-base') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(AlbertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.albert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), logits, (hidden_states), (attentions) \ No newline at end of file From c110c41fdb3b363528336ceda3b9fb46f026ad9c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 21:59:40 +0000 Subject: [PATCH 097/143] Run GLUE and remove LAMB --- examples/run_glue.py | 14 ++++-- transformers/optimization.py | 93 ------------------------------------ 2 files changed, 10 insertions(+), 97 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 527e440075..550a0b8175 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -47,7 +47,11 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForSequenceClassification, - DistilBertTokenizer) + DistilBertTokenizer, + AlbertConfig, + AlbertForSequenceClassification, + AlbertTokenizer, + ) from transformers import AdamW, get_linear_schedule_with_warmup @@ -66,7 +70,8 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer) } @@ -99,6 +104,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: @@ -317,7 +323,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): all_labels = torch.tensor([f.label for f in features], dtype=torch.long) elif output_mode == "regression": all_labels = torch.tensor([f.label for f in features], dtype=torch.float) - + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset @@ -361,7 +367,7 @@ def main(): parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.") + help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, diff --git a/transformers/optimization.py b/transformers/optimization.py index 90f3dbca9b..99e6cc75e4 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -167,96 +167,3 @@ class AdamW(Optimizer): p.data.add_(-group['lr'] * group['weight_decay'], p.data) return loss - - - -class Lamb(Optimizer): - """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training). - - Adapted from the huggingface/transformers ADAM optimizer - Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py - Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py - - - Parameters: - lr (float): learning rate. Default 1e-3. - betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) - eps (float): Adams epsilon. Default: 1e-6 - weight_decay (float): Weight decay. Default: 0.0 - """ - - def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): - if lr < 0.0: - raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) - if not 0.0 <= betas[0] < 1.0: - raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) - if not 0.0 <= betas[1] < 1.0: - raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) - if not 0.0 <= eps: - raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) - defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, - correct_bias=correct_bias) - super(Lamb, self).__init__(params, defaults) - - def step(self, closure=None): - """Performs a single optimization step. - - Arguments: - closure (callable, optional): A closure that reevaluates the model - and returns the loss. - """ - loss = None - if closure is not None: - loss = closure() - - for group in self.param_groups: - for p in group['params']: - if p.grad is None: - continue - grad = p.grad.data - if grad.is_sparse: - raise RuntimeError('LAMB does not support sparse gradients.') - - state = self.state[p] - - # State initialization - if len(state) == 0: - state['step'] = 0 - # Exponential moving average of gradient values - state['exp_avg'] = torch.zeros_like(p.data) - # Exponential moving average of squared gradient values - state['exp_avg_sq'] = torch.zeros_like(p.data) - - exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] - beta1, beta2 = group['betas'] - - state['step'] += 1 - - # Decay the first and second moment running average coefficient - # In-place operations to update the averages at the same time - exp_avg.mul_(beta1).add_(1.0 - beta1, grad) - exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) - denom = exp_avg_sq.sqrt().add_(group['eps']) - - - # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py - step_size = group['lr'] - weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) - - adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) - if group['weight_decay'] != 0: - adam_step.add_(group['weight_decay'], p.data) - - adam_norm = adam_step.pow(2).sum().sqrt() - if weight_norm == 0 or adam_norm == 0: - trust_ratio = 1 - else: - trust_ratio = weight_norm / adam_norm - - - state['weight_norm'] = weight_norm - state['adam_norm'] = adam_norm - state['trust_ratio'] = trust_ratio - - p.data.add_(-step_size * trust_ratio, adam_step) - return loss From 70d99980ded1565c9e8efa2cd04c21572b664f2f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 4 Nov 2019 11:34:30 -0500 Subject: [PATCH 098/143] ALBERT-V2 --- transformers/configuration_albert.py | 12 ++++++---- ...lbert_original_tf_checkpoint_to_pytorch.py | 5 ++-- transformers/modeling_albert.py | 16 ++++++++----- transformers/tokenization_albert.py | 24 ++++++++++++------- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index c35426768f..de665c9b1c 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -18,10 +18,14 @@ from .configuration_utils import PretrainedConfig ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-config.json", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-config.json", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-config.json", } class AlbertConfig(PretrainedConfig): diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py index 5bbaab8c21..b6476b4fb6 100644 --- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -26,9 +26,10 @@ from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_alb import logging logging.basicConfig(level=logging.INFO) -def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_file, pytorch_dump_path): # Initialise PyTorch model - config = AlbertConfig.from_json_file(bert_config_file) + config = AlbertConfig.from_json_file(albert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = AlbertForMaskedLM(config) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index bba6767079..51cb0a6d23 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -30,10 +30,14 @@ logger = logging.getLogger(__name__) ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-pytorch_model.bin", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-pytorch_model.bin", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-pytorch_model.bin", } @@ -538,8 +542,8 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): Examples:: - tokenizer = AlbertTokenizer.from_pretrained('albert-base') - model = AlbertForSequenceClassification.from_pretrained('albert-base') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForSequenceClassification.from_pretrained('albert-base-v2') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index acf67c1154..2f9af0b0bc 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -29,18 +29,26 @@ VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-spiece.model", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-spiece.model", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-spiece.model", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - 'albert-base': 512, - 'albert-large': 512, - 'albert-xlarge': 512, - 'albert-xxlarge': 512, + 'albert-base-v1': 512, + 'albert-large-v1': 512, + 'albert-xlarge-v1': 512, + 'albert-xxlarge-v1': 512, + 'albert-base-v2': 512, + 'albert-large-v2': 512, + 'albert-xlarge-v2': 512, + 'albert-xxlarge-v2': 512, } SPIECE_UNDERLINE = u'▁' From 4374eaea786399a948b4b34d39fc614ded5e1de6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 6 Nov 2019 20:47:42 +0000 Subject: [PATCH 099/143] ALBERT for SQuAD --- examples/run_squad.py | 12 +++-- transformers/__init__.py | 1 + transformers/modeling_albert.py | 93 ++++++++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 6 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 69088d73c3..59683c0668 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -43,7 +43,8 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLMTokenizer, XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer, - DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer, + AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) from transformers import AdamW, get_linear_schedule_with_warmup @@ -65,7 +66,8 @@ MODEL_CLASSES = { 'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) } def set_seed(args): @@ -128,7 +130,7 @@ def train(args, train_dataset, model, tokenizer): logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) logger.info(" Total optimization steps = %d", t_total) - global_step = 0 + global_step = 1 tr_loss, logging_loss = 0.0, 0.0 model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) @@ -537,7 +539,7 @@ def main(): torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) # Load a trained model and vocabulary that you have fine-tuned - model = model_class.from_pretrained(args.output_dir) + model = model_class.from_pretrained(args.output_dir, force_download=True) tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) model.to(args.device) @@ -555,7 +557,7 @@ def main(): for checkpoint in checkpoints: # Reload the model global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" - model = model_class.from_pretrained(checkpoint) + model = model_class.from_pretrained(checkpoint, force_download=True) model.to(args.device) # Evaluate diff --git a/transformers/__init__.py b/transformers/__init__.py index 51995942ce..81e659329d 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -108,6 +108,7 @@ if is_torch_available(): from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + AlbertForQuestionAnswering, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 51cb0a6d23..2540218e69 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -586,4 +586,95 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) outputs = (loss,) + outputs - return outputs # (loss), logits, (hidden_states), (attentions) \ No newline at end of file + return outputs # (loss), logits, (hidden_states), (attentions) + + + +@add_start_docstrings("""Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForQuestionAnswering(AlbertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForQuestionAnswering.from_pretrained('albert-base-v2') + question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]" + input_ids = tokenizer.encode(input_text) + token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] + start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids])) + all_tokens = tokenizer.convert_ids_to_tokens(input_ids) + print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])) + # a nice puppet + + + """ + def __init__(self, config): + super(AlbertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.albert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) From abb23a78bab17ec09dde4635c32f4aa21c15fa83 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:09:16 +0000 Subject: [PATCH 100/143] Head pruning for ALBERT --- transformers/modeling_albert.py | 42 ++++++++++++++++++++++ transformers/tests/modeling_albert_test.py | 12 ++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 2540218e69..89ece4b61e 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -145,6 +145,29 @@ class AlbertAttention(BertSelfAttention): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.pruned_heads = set() + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.dense = prune_linear_layer(self.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) mixed_key_layer = self.key(input_ids) @@ -409,6 +432,25 @@ class AlbertModel(AlbertPreTrainedModel): self.embeddings.word_embeddings = new_embeddings return self.embeddings.word_embeddings + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + ALBERT has a different architecture in that its layers are shared across groups, which then has inner groups. + If an ALBERT model has 12 hidden layers and 2 hidden groups, with two inner groups, there + is a total of 4 different layers. + + These layers are flattened: the indices [0,1] correspond to the two inner groups of the first hidden layer, + while [2,3] correspond to the two inner groups of the second hidden layer. + + Any layer with in index other than [0,1,2,3] will result in an error. + See base class PreTrainedModel for more information about head pruning + """ + for layer, heads in heads_to_prune.items(): + group_idx = int(layer / self.config.inner_group_num) + inner_group_idx = int(layer - group_idx * self.config.inner_group_num) + self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 46a2eeb729..979e0488eb 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -35,7 +35,6 @@ else: class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () - test_pruning = False test_head_masking = False class AlbertModelTester(object): @@ -49,9 +48,10 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, - hidden_size=32, - num_hidden_layers=5, - num_attention_heads=4, + hidden_size=36, + num_hidden_layers=6, + num_hidden_groups=6, + num_attention_heads=6, intermediate_size=37, hidden_act="gelu", hidden_dropout_prob=0.1, @@ -86,6 +86,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): self.num_labels = num_labels self.num_choices = num_choices self.scope = scope + self.num_hidden_groups = num_hidden_groups def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) @@ -117,7 +118,8 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): attention_probs_dropout_prob=self.attention_probs_dropout_prob, max_position_embeddings=self.max_position_embeddings, type_vocab_size=self.type_vocab_size, - initializer_range=self.initializer_range) + initializer_range=self.initializer_range, + num_hidden_groups=self.num_hidden_groups) return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels From 16263f9685eaf459408f33c9790a967012b93fa5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:29:29 +0000 Subject: [PATCH 101/143] Headmasking --- transformers/modeling_albert.py | 11 ++++++----- transformers/tests/modeling_albert_test.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 89ece4b61e..6682930d89 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -224,7 +224,7 @@ class AlbertLayer(nn.Module): self.activation = ACT2FN[config.hidden_act] def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask) + attention_output = self.attention(hidden_states, attention_mask, head_mask) ffn_output = self.ffn(attention_output[0]) ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) @@ -245,8 +245,8 @@ class AlbertLayerGroup(nn.Module): layer_hidden_states = () layer_attentions = () - for albert_layer in self.albert_layers: - layer_output = albert_layer(hidden_states, attention_mask, head_mask) + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer(hidden_states, attention_mask, head_mask[layer_index]) hidden_states = layer_output[0] if self.output_attentions: @@ -283,7 +283,8 @@ class AlbertTransformer(nn.Module): for layer_idx in range(self.config.num_hidden_layers): group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) hidden_states = layer_group_output[0] @@ -544,7 +545,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): - outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask) sequence_outputs = outputs[0] prediction_scores = self.predictions(sequence_outputs) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 979e0488eb..466f473332 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -35,7 +35,6 @@ else: class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () - test_head_masking = False class AlbertModelTester(object): From 9d5c49546fedb3d52f76f97f4043a07e08ded918 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:32:52 +0000 Subject: [PATCH 102/143] Tests for AlbertForQuestionAnswering AlbertForSequenceClassification --- transformers/tests/modeling_albert_test.py | 45 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 466f473332..da87709df1 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -26,7 +26,9 @@ from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester if is_torch_available(): - from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM) + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM, + AlbertForSequenceClassification, AlbertForQuestionAnswering, + ) from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -157,6 +159,39 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): [self.batch_size, self.seq_length, self.vocab_size]) self.check_loss_output(result) + def create_and_check_albert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForQuestionAnswering(config=config) + model.eval() + loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, + start_positions=sequence_labels, end_positions=sequence_labels) + result = { + "loss": loss, + "start_logits": start_logits, + "end_logits": end_logits, + } + self.parent.assertListEqual( + list(result["start_logits"].size()), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].size()), + [self.batch_size, self.seq_length]) + self.check_loss_output(result) + + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = AlbertForSequenceClassification(config) + model.eval() + loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) + result = { + "loss": loss, + "logits": logits, + } + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.num_labels]) + self.check_loss_output(result) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() @@ -180,6 +215,14 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification(*config_and_inputs) + @pytest.mark.slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" From d9daad98c744e115bfb425f316a5e7d4f405a9a5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 19:55:43 +0000 Subject: [PATCH 103/143] Re-ordering of group_idx/layer_idx + Python 2 tests --- transformers/modeling_albert.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 6682930d89..640af537d0 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -281,11 +281,17 @@ class AlbertTransformer(nn.Module): if self.output_hidden_states: all_hidden_states = (hidden_states,) - for layer_idx in range(self.config.num_hidden_layers): - group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + # Index of the layer inside the group + layer_idx = int(i - group_idx * layers_per_group) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) - hidden_states = layer_group_output[0] if self.output_attentions: From f6f382532bf40a5869dc14e3eefa451646c19ded Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 23:40:45 +0000 Subject: [PATCH 104/143] ALBERT in TF2 --- transformers/__init__.py | 6 +- .../convert_pytorch_checkpoint_to_tf2.py | 13 +- transformers/modeling_tf_albert.py | 723 ++++++++++++++++++ 3 files changed, 736 insertions(+), 6 deletions(-) create mode 100644 transformers/modeling_tf_albert.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 81e659329d..a409ef772e 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -58,8 +58,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_albert import AlbertConfig, ALBERT -from .configuration_albert import AlbertConfig +from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling @@ -169,6 +168,9 @@ if is_tf_available(): TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + # TF 2.0 <=> PyTorch conversion utilities from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, load_pytorch_checkpoint_in_tf2_model, diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index e673b77dcc..d1776e9c14 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -33,7 +33,8 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + AlbertConfig, TFAlbertForMaskedLM, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch @@ -46,7 +47,8 @@ if is_torch_available(): OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) else: (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, @@ -56,7 +58,8 @@ else: OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) = ( + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) = ( None, None, None, None, None, None, None, None, @@ -65,6 +68,7 @@ else: None, None, None, None, None, None, None, None, + None, None, None, None) @@ -85,7 +89,8 @@ MODEL_CLASSES = { 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), - 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'albert': (AlbertConfig, TFAlbertForMaskedLM, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True): diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py new file mode 100644 index 0000000000..8861b7add8 --- /dev/null +++ b/transformers/modeling_tf_albert.py @@ -0,0 +1,723 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 ALBERT model. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_albert import AlbertConfig +from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_bert import ACT2FN, TFBertSelfAttention +from .file_utils import add_start_docstrings + +import logging + +logger = logging.getLogger(__name__) + +TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + # TODO FILL THAT UP +} + + +class TFAlbertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config, **kwargs): + super(TFAlbertEmbeddings, self).__init__(**kwargs) + + self.config = config + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.config.vocab_size, self.config.embedding_size], + initializer=get_initializer(self.config.initializer_range)) + super(TFAlbertEmbeddings, self).build(input_shape) + + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + words_embeddings = tf.gather(self.word_embeddings, input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings, training=training) + return embeddings + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, embedding_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + print(inputs.shape) + + x = tf.reshape(inputs, [-1, self.config.embedding_size]) + + print(x.shape, self.word_embeddings) + + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + + print([batch_size, length, self.config.vocab_size]) + return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) + + +class TFAlbertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int( + config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='value') + + self.dropout = tf.keras.layers.Dropout( + config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape( + x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = tf.shape(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFAlbertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + return outputs + + +class TFAlbertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFAlbertAttention(TFBertSelfAttention): + def __init__(self, config, **kwargs): + super(TFAlbertAttention, self).__init__(config, **kwargs) + + self.hidden_size = config.hidden_size + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + batch_size = tf.shape(input_tensor)[0] + mixed_query_layer = self.query(input_tensor) + mixed_key_layer = self.key(input_tensor) + mixed_value_layer = self.value(input_tensor) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + self_outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + + hidden_states = self_outputs[0] + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + attention_output = self.LayerNorm(hidden_states + input_tensor) + + # add attentions if we output them + outputs = (attention_output,) + self_outputs[1:] + return outputs + + +class TFAlbertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayer, self).__init__(**kwargs) + self.attention = TFAlbertAttention(config, name='attention') + + self.ffn = tf.keras.layers.Dense(config.intermediate_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn') + + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.ffn_output = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn_output') + self.full_layer_layer_norm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='full_layer_layer_norm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention( + [hidden_states, attention_mask, head_mask], training=training) + ffn_output = self.ffn(attention_outputs[0]) + ffn_output = self.activation(ffn_output) + ffn_output = self.ffn_output(ffn_output) + + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.full_layer_layer_norm( + ffn_output + attention_outputs[0]) + + # add attentions if we output them + outputs = (hidden_states,) + attention_outputs[1:] + return outputs + + +class TFAlbertLayerGroup(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayerGroup, self).__init__(**kwargs) + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.albert_layers = [TFAlbertLayer(config, name="albert_layers_._{}".format( + i)) for i in range(config.inner_group_num)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + layer_hidden_states = () + layer_attentions = () + + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer( + [hidden_states, attention_mask, head_mask[layer_index]], training=training) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + # last-layer hidden state, (layer hidden states), (layer attentions) + return outputs + + +class TFAlbertTransformer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertTransformer, self).__init__(**kwargs) + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='embedding_hidden_mapping_in') + self.albert_layer_groups = [TFAlbertLayerGroup( + config, name="albert_layer_groups_._{}".format(i)) for i in range(config.num_hidden_groups)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group + layers_per_group = int( + self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int( + i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + layer_group_output = self.albert_layer_groups[group_idx]( + [hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]], training=training) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[-1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + + # last-layer hidden state, (all hidden states), (all attentions) + return outputs + + +class TFAlbertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + +class TFAlbertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, input_embeddings, **kwargs): + super(TFAlbertMLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + + self.dense = tf.keras.layers.Dense(config.embedding_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.input_embeddings = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + super(TFAlbertMLMHead, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias + return hidden_states + + +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, ALBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Albert Model transformer outputing raw hidden-states without any specific head on top.", + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertModel(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Albert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertModel + + tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') + model = TFAlbertModel.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + + def __init__(self, config, **kwargs): + super(TFAlbertModel, self).__init__(config, **kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFAlbertEmbeddings(config, name="embeddings") + self.encoder = TFAlbertTransformer(config, name="encoder") + self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), activation='tanh', name='pooler') + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings( + [input_ids, position_ids, token_type_ids], training=training) + encoder_outputs = self.encoder( + [embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] + # sequence_output, pooled_output, (hidden_states), (attentions) + return outputs + + +@add_start_docstrings("""Albert Model with a `language modeling` head on top. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForMaskedLM + + tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') + model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + prediction_scores = outputs[0] + + """ + + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForMaskedLM, self).__init__(config, *inputs, **kwargs) + + self.albert = TFAlbertModel(config, name='albert') + self.predictions = TFAlbertMLMHead( + config, self.albert.embeddings, name='predictions') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + sequence_output = outputs[0] + prediction_scores = self.predictions( + sequence_output, training=kwargs.get('training', False)) + + # Add hidden states and attention if they are here + outputs = (prediction_scores,) + outputs[2:] + + return outputs # prediction_scores, (hidden_states), (attentions) From 7bddbf5961f35d03db5151a4747085f7a57d7ff7 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 23:50:05 +0000 Subject: [PATCH 105/143] TFAlbertForSequenceClassification --- transformers/modeling_tf_albert.py | 66 ++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 8861b7add8..410d83ff78 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -479,16 +479,15 @@ class TFAlbertMLMHead(tf.keras.layers.Layer): ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in - `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ - by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer - pre-trained using a combination of masked language modeling objective and next sentence prediction - on a large corpus comprising the Toronto Book Corpus and Wikipedia. + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. - .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: - https://arxiv.org/abs/1810.04805 + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model @@ -695,8 +694,8 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): import tensorflow as tf from transformers import AlbertTokenizer, TFAlbertForMaskedLM - tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') - model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForMaskedLM.from_pretrained('albert-base-v2') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -721,3 +720,54 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): outputs = (prediction_scores,) + outputs[2:] return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForSequenceClassification + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForSequenceClassification.from_pretrained('albert-base-v2') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.albert = TFAlbertModel(config, name='albert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) \ No newline at end of file From b18509c2085dce16e655dd86b7ea0129335da4e1 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 8 Nov 2019 00:12:21 +0000 Subject: [PATCH 106/143] Tests for ALBERT in TF2 + fixes --- transformers/__init__.py | 1 + transformers/modeling_tf_albert.py | 18 +- transformers/tests/modeling_tf_albert_test.py | 229 ++++++++++++++++++ 3 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 transformers/tests/modeling_tf_albert_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index a409ef772e..baf430c17b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -169,6 +169,7 @@ if is_tf_available(): TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # TF 2.0 <=> PyTorch conversion utilities diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 410d83ff78..a3f183b192 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -126,16 +126,8 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): """ batch_size = tf.shape(inputs)[0] length = tf.shape(inputs)[1] - - print(inputs.shape) - x = tf.reshape(inputs, [-1, self.config.embedding_size]) - - print(x.shape, self.word_embeddings) - logits = tf.matmul(x, self.word_embeddings, transpose_b=True) - - print([batch_size, length, self.config.vocab_size]) return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) @@ -460,20 +452,24 @@ class TFAlbertMLMHead(tf.keras.layers.Layer): # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. - self.input_embeddings = input_embeddings + self.decoder = input_embeddings def build(self, input_shape): self.bias = self.add_weight(shape=(self.vocab_size,), initializer='zeros', trainable=True, name='bias') + self.decoder_bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='decoder/bias') super(TFAlbertMLMHead, self).build(input_shape) def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) - hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = self.decoder(hidden_states, mode="linear") + self.decoder_bias hidden_states = hidden_states + self.bias return hidden_states @@ -666,7 +662,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): [embedding_output, extended_attention_mask, head_mask], training=training) sequence_output = encoder_outputs[0] - pooled_output = self.pooler(sequence_output) + pooled_output = self.pooler(sequence_output[:, 0]) # add hidden_states and attentions if they are here outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py new file mode 100644 index 0000000000..85fc62f34f --- /dev/null +++ b/transformers/tests/modeling_tf_albert_test.py @@ -0,0 +1,229 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from transformers import AlbertConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from transformers.modeling_tf_albert import (TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = ( + TFAlbertModel, + TFAlbertForMaskedLM, + TFAlbertForSequenceClassification + ) if is_tf_available() else () + + class TFAlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor( + [self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor( + [self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor( + [self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor( + [self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor( + [self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertModel(config=config) + # inputs = {'input_ids': input_ids, + # 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids} + # sequence_output, pooled_output = model(**inputs) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [ + self.batch_size, self.hidden_size]) + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFAlbertForSequenceClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, + 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFAlbertModelTest.TFAlbertModelTester(self) + self.config_tester = ConfigTester( + self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm( + *config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification( + *config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['albert-base-uncased']: + model = TFAlbertModel.from_pretrained( + model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() From c9cb7f8a0fbe784665b00bfdca6bfc54ad10d5f5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 11 Nov 2019 15:12:54 -0500 Subject: [PATCH 107/143] Torch 1.1.0 compatibility + FP16 O1 + TF checkpoints Co-authored-by: wassname --- transformers/modeling_albert.py | 4 ++-- transformers/modeling_tf_albert.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 640af537d0..ff20ca78dc 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -203,8 +203,8 @@ class AlbertAttention(BertSelfAttention): # Should find a better way to do this - w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) - b = self.dense.bias + w = self.dense.weight.t().view(self.num_attention_heads, self.attention_head_size, self.hidden_size).to(context_layer.dtype) + b = self.dense.bias.to(context_layer.dtype) projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b projected_context_layer_dropout = self.dropout(projected_context_layer) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index a3f183b192..ee8712eb28 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -36,7 +36,14 @@ import logging logger = logging.getLogger(__name__) TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - # TODO FILL THAT UP + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-tf_model.h5", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-tf_model.h5", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-tf_model.h5", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-tf_model.h5", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-tf_model.h5", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-tf_model.h5", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-tf_model.h5", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-tf_model.h5", } From f873b55e433e1c9ef364e597ad1d4662db4bc038 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 26 Nov 2019 10:28:41 -0500 Subject: [PATCH 108/143] Warning for ALBERT-v2 models --- transformers/modeling_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d51eefab58..5d51caa951 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -315,6 +315,10 @@ class PreTrainedModel(nn.Module): model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ + if "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path: + logger.warning("There is currently an upstream reproducibility issue with ALBERT v2 models. Please see " + + "https://github.com/google-research/google-research/issues/119 for more information.") + config = kwargs.pop('config', None) state_dict = kwargs.pop('state_dict', None) cache_dir = kwargs.pop('cache_dir', None) From c536c2a4809a51356fd50ff62a80955fea79f790 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 26 Nov 2019 11:22:52 -0500 Subject: [PATCH 109/143] ALBERT Input Embeds --- transformers/modeling_albert.py | 77 ++++++++++++++++++++++-------- transformers/modeling_tf_albert.py | 45 ++++++++++++----- 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index ff20ca78dc..7882356d24 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -433,6 +433,12 @@ class AlbertModel(AlbertPreTrainedModel): self.init_weights() + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + def _resize_token_embeddings(self, new_num_tokens): old_embeddings = self.embeddings.word_embeddings new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) @@ -457,12 +463,24 @@ class AlbertModel(AlbertPreTrainedModel): inner_group_idx = int(layer - group_idx * self.config.inner_group_num) self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None): + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: - attention_mask = torch.ones_like(input_ids) + attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility @@ -477,7 +495,8 @@ class AlbertModel(AlbertPreTrainedModel): else: head_mask = [None] * self.config.num_hidden_layers - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds) encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) @@ -549,9 +568,19 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): self._tie_or_clone_weights(self.predictions.decoder, self.albert.embeddings.word_embeddings) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - masked_lm_labels=None): - outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask) + def get_output_embeddings(self): + return self.predictions.decoder + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None, inputs_embeds=None): + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) sequence_outputs = outputs[0] prediction_scores = self.predictions(sequence_outputs) @@ -609,14 +638,17 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): - outputs = self.albert(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) pooled_output = outputs[1] @@ -692,14 +724,17 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - start_positions=None, end_positions=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None, start_positions=None, end_positions=None): - outputs = self.albert(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) sequence_output = outputs[0] diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index ee8712eb28..ee122205b9 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -107,19 +107,25 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): def _embedding(self, inputs, training=False): """Applies embedding based on inputs tensor.""" - input_ids, position_ids, token_type_ids = inputs + input_ids, position_ids, token_type_ids, inputs_embeds = inputs - seq_length = tf.shape(input_ids)[1] + if input_ids is not None: + input_shape = tf.shape(input_ids) + else: + input_shape = tf.shape(inputs_embeds)[:-1] + + seq_length = input_shape[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) - words_embeddings = tf.gather(self.word_embeddings, input_ids) + if inputs_embeds is None: + inputs_embeds = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = inputs_embeds + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) embeddings = self.dropout(embeddings, training=training) return embeddings @@ -603,6 +609,9 @@ class TFAlbertModel(TFAlbertPreTrainedModel): self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( config.initializer_range), activation='tanh', name='pooler') + def get_input_embeddings(self): + return self.embeddings + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -613,28 +622,39 @@ class TFAlbertModel(TFAlbertPreTrainedModel): """ raise NotImplementedError - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.shape + elif inputs_embeds is not None: + input_shape = inputs_embeds.shape[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) + attention_mask = tf.fill(input_shape, 1) if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] @@ -664,7 +684,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): # head_mask = tf.constant([0] * self.num_hidden_layers) embedding_output = self.embeddings( - [input_ids, position_ids, token_type_ids], training=training) + [input_ids, position_ids, token_type_ids, inputs_embeds], training=training) encoder_outputs = self.encoder( [embedding_output, extended_attention_mask, head_mask], training=training) @@ -712,6 +732,9 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): self.predictions = TFAlbertMLMHead( config, self.albert.embeddings, name='predictions') + def get_output_embeddings(self): + return self.albert.embeddings + def call(self, inputs, **kwargs): outputs = self.albert(inputs, **kwargs) From bdfe21ab242c2175972f53ca6b4af29ada02a2bb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 12:54:36 -0500 Subject: [PATCH 110/143] Change param order for consistency --- transformers/modeling_albert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 7882356d24..5b7b2d3900 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -571,8 +571,8 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def get_output_embeddings(self): return self.predictions.decoder - def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - masked_lm_labels=None, inputs_embeds=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, + masked_lm_labels=None): outputs = self.albert( input_ids=input_ids, attention_mask=attention_mask, From f2f329408db66285fd59e6628ca394381bb7f94e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 12:59:28 -0500 Subject: [PATCH 111/143] Fix input embeddings --- transformers/tests/modeling_albert_test.py | 2 ++ transformers/tests/modeling_tf_albert_test.py | 2 ++ transformers/tests/modeling_tf_common_test.py | 7 ++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index da87709df1..976feff9db 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -49,6 +49,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, + embedding_size=16, hidden_size=36, num_hidden_layers=6, num_hidden_groups=6, @@ -73,6 +74,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index 85fc62f34f..fbd519b8f6 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -54,6 +54,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, + embedding_size=16, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, @@ -77,6 +78,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 2bb7cc9c5f..31a30766cf 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,9 +426,10 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) - # ^^ In our TF models, the input_embeddings can take slightly different forms, - # so we try two of them and fall back to just synthetically creating a dummy tensor of ones. + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From ae98d4599179b299563b679dd33f8a86da12980d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:12:44 -0500 Subject: [PATCH 112/143] Release: v2.2.0 --- .circleci/deploy.sh | 1 + README.md | 2 +- deploy_multi_version_doc.sh | 22 ++++++++++++++++++++++ docs/source/conf.py | 2 +- setup.py | 2 +- transformers/__init__.py | 2 +- 6 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 deploy_multi_version_doc.sh diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 98151963d8..c4b802c9e9 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,3 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 diff --git a/README.md b/README.md index 52e74a6f80..bb6a8eaa1a 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation](https://huggingface.co/transformers/) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more | +| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more | ## Installation diff --git a/deploy_multi_version_doc.sh b/deploy_multi_version_doc.sh new file mode 100644 index 0000000000..bd567213eb --- /dev/null +++ b/deploy_multi_version_doc.sh @@ -0,0 +1,22 @@ +cd docs + +function deploy_doc(){ + echo "Creating doc at commit $1 and pushing to folder $2" + git checkout $1 + if [ ! -z "$2" ] + then + echo "Pushing version" $2 + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 + else + echo "Pushing master" + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir + fi +} + +deploy_doc "master" +deploy_doc "b33a385" v1.0.0 +deploy_doc "fe02e45" v1.1.0 +deploy_doc "89fd345" v1.2.0 +deploy_doc "fc9faa8" v2.0.0 +deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 00c020ab39..f762a89cd2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.1.1' +release = u'2.2.0' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index f49aee68d4..d8dcf7b898 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ from setuptools import find_packages, setup setup( name="transformers", - version="2.1.1", + version="2.2.0", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", diff --git a/transformers/__init__.py b/transformers/__init__.py index baf430c17b..dff2479d4b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.1.1" +__version__ = "2.2.0" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. From b6321452738925983ba1306fb5396982d64f408a Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Tue, 26 Nov 2019 14:27:15 -0500 Subject: [PATCH 113/143] Update master documentation link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb6a8eaa1a..67fb80aecf 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more | +| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation From cf62bdc962c53d9fb7a5820217f1bf844bb6da3b Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 14:37:32 -0500 Subject: [PATCH 114/143] Improve test protocol for inputs_embeds in TF cc @lysandrejik --- transformers/tests/modeling_tf_common_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 31a30766cf..232991915c 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,10 +426,15 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - if hasattr(self.model_tester, "embedding_size"): - x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32) - else: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) + x = wte([input_ids, None, None, None], mode="embedding") + # ^^ In our TF models, the input_embeddings can take slightly different forms, + # so we try a few of them. + # We used to fall back to just synthetically creating a dummy tensor of ones: + # + # if hasattr(self.model_tester, "embedding_size"): + # x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + # else: + # x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From 8742baa53136b906e392fdae57f1c191d1b2370e Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 14:39:47 -0500 Subject: [PATCH 115/143] Improve test protocol for inputs_embeds in TF --- transformers/tests/modeling_tf_common_test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 232991915c..ea8cd1aecd 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,15 +426,17 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - x = wte([input_ids, None, None, None], mode="embedding") + try: + x = wte([input_ids, None, None, None], mode="embedding") + except: + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) # ^^ In our TF models, the input_embeddings can take slightly different forms, # so we try a few of them. # We used to fall back to just synthetically creating a dummy tensor of ones: # - # if hasattr(self.model_tester, "embedding_size"): - # x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) - # else: - # x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From 668aac45d206bc0c8ca95e7a8b565d1ff701da7f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:52:42 -0500 Subject: [PATCH 116/143] Pretrained models --- docs/source/pretrained_models.rst | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index b0a578fd80..d017a3be6c 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -159,5 +159,38 @@ Here is the full list of the currently provided pretrained models together with | | | | CamemBERT using the BERT-base architecture | | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ + .. `__ From 7c6000e412288ddce7d0dbc4f4d9a12f49d28690 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:55:29 -0500 Subject: [PATCH 117/143] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index c4b802c9e9..73960cbe99 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "f2f3294" v2.2.0 +deploy_doc "668aac4" v2.2.0 From ee4647bd5c1343304f7fdf20f06db1fc2524f892 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:10:51 -0500 Subject: [PATCH 118/143] CamemBERT & ALBERT doc --- docs/source/index.rst | 5 ++ docs/source/model_doc/albert.rst | 71 +++++++++++++++++++++++++++++ docs/source/model_doc/camembert.rst | 50 ++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 docs/source/model_doc/albert.rst create mode 100644 docs/source/model_doc/camembert.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 4cd1f48ba8..55ead33b4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -47,6 +47,9 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 6. `XLM `_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `_ by Guillaume Lample and Alexis Conneau. 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. `DistilBERT `_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 `_. +9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. +10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. +11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. .. toctree:: :maxdepth: 2 @@ -89,3 +92,5 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train model_doc/roberta model_doc/distilbert model_doc/ctrl + model_doc/camembert + model_doc/albert diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst new file mode 100644 index 0000000000..cf52f35eb6 --- /dev/null +++ b/docs/source/model_doc/albert.rst @@ -0,0 +1,71 @@ +ALBERT +---------------------------------------------------- + +``AlbrtConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertConfig + :members: + + +``AlbertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertTokenizer + :members: + + +``AlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertModel + :members: + + +``AlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForMaskedLM + :members: + + +``AlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForSequenceClassification + :members: + + +``AlbertForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForQuestionAnswering + :members: + + +``TFAlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertModel + :members: + + +``TFBertForPreTraining`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFBertForPreTraining + :members: + + +``TFAlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForMaskedLM + :members: + + +``TFAlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForSequenceClassification + :members: diff --git a/docs/source/model_doc/camembert.rst b/docs/source/model_doc/camembert.rst new file mode 100644 index 0000000000..82ca9de945 --- /dev/null +++ b/docs/source/model_doc/camembert.rst @@ -0,0 +1,50 @@ +CamemBERT +---------------------------------------------------- + +``CamembertConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertConfig + :members: + + +``CamembertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertTokenizer + :members: + + +``CamembertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertModel + :members: + + +``CamembertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMaskedLM + :members: + + +``CamembertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForSequenceClassification + :members: + + +``CamembertForMultipleChoice`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMultipleChoice + :members: + + +``CamembertForTokenClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForTokenClassification + :members: From 44b82c777f0a4e173fa9c68de630a63fbac1b946 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:15:11 -0500 Subject: [PATCH 119/143] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 73960cbe99..8081153b9f 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "668aac4" v2.2.0 +deploy_doc "ee4647b" v2.2.0 From cf26a0c85e0520d3e3770c9ad8dce94318ea3440 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:40:03 -0500 Subject: [PATCH 120/143] Fix pretrained models table --- README.md | 1 + docs/source/pretrained_models.rst | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 67fb80aecf..dd2c865036 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. +11. **[ALBERT](https://github.com/google-research/google-research/tree/master/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index d017a3be6c..a6bf508d18 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -162,31 +162,31 @@ Here is the full list of the currently provided pretrained models together with | ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | From ce02550d50f2074d54d58b37cbc9845d9a159818 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:47:02 -0500 Subject: [PATCH 121/143] Fix pretrained models table --- docs/source/pretrained_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index a6bf508d18..a312346df7 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -190,7 +190,7 @@ Here is the full list of the currently provided pretrained models together with | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ .. `__ From cc7968227e08858df4a5c618c739e1a3ca050196 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:52:25 -0500 Subject: [PATCH 122/143] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 8081153b9f..61b9f90909 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "ee4647b" v2.2.0 +deploy_doc "ce02550" v2.2.0 From 361620954acf16b27727d763a591257b03f90b5d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 27 Nov 2019 10:11:37 -0500 Subject: [PATCH 123/143] Remove TFBertForPreTraining from ALBERT doc --- docs/source/model_doc/albert.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst index cf52f35eb6..92970c9328 100644 --- a/docs/source/model_doc/albert.rst +++ b/docs/source/model_doc/albert.rst @@ -50,13 +50,6 @@ ALBERT :members: -``TFBertForPreTraining`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: transformers.TFBertForPreTraining - :members: - - ``TFAlbertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 45d767297a78697e4ff75ec3d83bdf35e43e4fff Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 27 Nov 2019 10:12:20 -0500 Subject: [PATCH 124/143] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 61b9f90909..a32581baef 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "ce02550" v2.2.0 +deploy_doc "3616209" v2.2.0 From 88b317739fe56888528c857fc8e90967148a0051 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 27 Nov 2019 09:58:38 +0000 Subject: [PATCH 125/143] Fix issue: #1962, input's shape seem to cause error in 2.2.0 version tf_albert_model --- transformers/modeling_tf_albert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index ee122205b9..b2bf66f750 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -645,7 +645,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: - input_shape = input_ids.shape + input_shape = tf.shape(input_ids) elif inputs_embeds is not None: input_shape = inputs_embeds.shape[:-1] else: From de2696f68e20019fef3a5e1b54de10351abb4145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Tue, 26 Nov 2019 16:30:27 +0100 Subject: [PATCH 126/143] suggest to track repo w/ https rather than ssh --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 136ef8df81..8228dd59d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ Follow these steps to start contributing: ```bash $ git clone git@github.com:/transformers.git $ cd transformers - $ git remote add upstream git@github.com:huggingface/transformers.git + $ git remote add upstream https://github.com/huggingface/transformers.git ``` 3. Create a new branch to hold your development changes: From b5d884d25c96069962030b9e4c9dfb4b1f2f6fa0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 27 Nov 2019 11:05:18 -0500 Subject: [PATCH 127/143] Uniformize #1952 --- README.md | 2 +- examples/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd2c865036..dd20b80590 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Examples are included in the repository but are not shipped with the library. Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: ```bash -git clone git@github.com:huggingface/transformers +git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . ``` diff --git a/examples/README.md b/examples/README.md index 0c57990ea7..e109a12171 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ similar API between the different models. To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: ```bash -git clone git@github.com:huggingface/transformers +git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . ``` From 71f71ddb3e04aa18aaa1e3633a2881493e67b438 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 11:50:42 -0400 Subject: [PATCH 128/143] run_xnli + utils_xnli --- examples/run_xnli.py | 534 +++++++++++++++++++++++++++++++++++++++++ examples/utils_xnli.py | 93 +++++++ 2 files changed, 627 insertions(+) create mode 100644 examples/run_xnli.py create mode 100644 examples/utils_xnli.py diff --git a/examples/run_xnli.py b/examples/run_xnli.py new file mode 100644 index 0000000000..ee37296832 --- /dev/null +++ b/examples/run_xnli.py @@ -0,0 +1,534 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning multi-lingual models on XNLI (Bert, XLM). + Adapted from `examples/run_glue.py`""" + +from __future__ import absolute_import, division, print_function + +import argparse +import glob +import logging +import os +import random + +import numpy as np +import torch +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + TensorDataset) +from torch.utils.data.distributed import DistributedSampler + +try: + from torch.utils.tensorboard import SummaryWriter +except: + from tensorboardX import SummaryWriter + +from tqdm import tqdm, trange + +from transformers import (WEIGHTS_NAME, + BertConfig, BertForSequenceClassification, BertTokenizer, + XLMConfig, XLMForSequenceClassification, XLMTokenizer, + DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + +from transformers import AdamW, WarmupLinearSchedule + +from utils_xnli import xnli_compute_metrics as compute_metrics +from utils_xnli import xnli_output_modes as output_modes +from utils_xnli import xnli_processors as processors + +from transformers import glue_convert_examples_to_features as convert_examples_to_features + +logger = logging.getLogger(__name__) + +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ()) + +MODEL_CLASSES = { + 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), + 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), + # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) +} + + +def set_seed(args): + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.n_gpu > 0: + torch.cuda.manual_seed_all(args.seed) + + +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + if args.local_rank in [-1, 0]: + tb_writer = SummaryWriter() + + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + + if args.max_steps > 0: + t_total = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", t_total) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: + epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) + for step, batch in enumerate(epoch_iterator): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + tr_loss += loss.item() + if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if args.fp16: + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + # Log metrics + if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well + results = evaluate(args, model, tokenizer) + for key, value in results.items(): + tb_writer.add_scalar('eval_{}'.format(key), value, global_step) + tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) + tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + logging_loss = tr_loss + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + if args.tpu: + args.xla_model.optimizer_step(optimizer, barrier=True) + model.zero_grad() + global_step += 1 + + if args.max_steps > 0 and global_step > args.max_steps: + epoch_iterator.close() + break + if args.max_steps > 0 and global_step > args.max_steps: + train_iterator.close() + break + + if args.local_rank in [-1, 0]: + tb_writer.close() + + return global_step, tr_loss / global_step + + +def evaluate(args, model, tokenizer, prefix=""): + eval_task_names = (args.task_name,) + eval_outputs_dirs = (args.output_dir,) + + results = {} + for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): + eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) + + if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: + os.makedirs(eval_output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(eval_dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + out_label_ids = None + for batch in tqdm(eval_dataloader, desc="Evaluating"): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + tmp_eval_loss, logits = outputs[:2] + + eval_loss += tmp_eval_loss.mean().item() + nb_eval_steps += 1 + if preds is None: + preds = logits.detach().cpu().numpy() + out_label_ids = inputs['labels'].detach().cpu().numpy() + else: + preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) + out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0) + + eval_loss = eval_loss / nb_eval_steps + if args.output_mode == "classification": + preds = np.argmax(preds, axis=1) + elif args.output_mode == "regression": + preds = np.squeeze(preds) + result = compute_metrics(eval_task, preds, out_label_ids) + results.update(result) + + output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") + with open(output_eval_file, "w") as writer: + logger.info("***** Eval results {} *****".format(prefix)) + for key in sorted(result.keys()): + logger.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + return results + + +def load_and_cache_examples(args, task, tokenizer, evaluate=False): + if args.local_rank not in [-1, 0] and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + processor = processors[task](language=args.language, train_language=args.train_language) + output_mode = output_modes[task] + # Load data features from cache or dataset file + cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( + 'dev' if evaluate else 'train', + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length), + str(task), + str(args.train_language if (not evaluate and args.train_language is not None) else args.language))) + if os.path.exists(cached_features_file) and not args.overwrite_cache: + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", args.data_dir) + label_list = processor.get_labels() + examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + features = convert_examples_to_features(examples, + tokenizer, + label_list=label_list, + max_length=args.max_seq_length, + output_mode=output_mode, + pad_on_left=False, + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=0, + ) + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + + if args.local_rank == 0 and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + if output_mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) + else: + raise ValueError(f'No other `output_mode` for XNLI.') + + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) + return dataset + + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + parser.add_argument("--model_type", default=None, type=str, required=True, + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument("--model_name_or_path", default=None, type=str, required=True, + help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + parser.add_argument("--language", default=None, type=str, required=True, + help="Evaluation language. Also train language if `train_language` is set to None.") + parser.add_argument("--train_language", default=None, type=str, + help="Train language if is different of the evaluation language.") + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + + ## Other parameters + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--cache_dir", default="", type=str, + help="Where do you want to store the pre-trained models downloaded from s3") + parser.add_argument("--max_seq_length", default=128, type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.") + parser.add_argument("--do_train", action='store_true', + help="Whether to run training.") + parser.add_argument("--do_eval", action='store_true', + help="Whether to run eval on the dev set.") + parser.add_argument("--evaluate_during_training", action='store_true', + help="Rul evaluation during training at each logging step.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=3.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_steps", default=0, type=int, + help="Linear warmup over warmup_steps.") + + parser.add_argument('--logging_steps', type=int, default=50, + help="Log every X updates steps.") + parser.add_argument('--save_steps', type=int, default=50, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Avoid using CUDA when available") + parser.add_argument('--overwrite_output_dir', action='store_true', + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument('--tpu', action='store_true', + help="Whether to run on the TPU defined in the environment variables") + parser.add_argument('--tpu_ip_address', type=str, default='', + help="TPU IP address if none are set in the environment variables") + parser.add_argument('--tpu_name', type=str, default='', + help="TPU name if none are set in the environment variables") + parser.add_argument('--xrt_tpu_config', type=str, default='', + help="XRT TPU config if none are set in the environment variables") + + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument("--local_rank", type=int, default=-1, + help="For distributed training: local_rank") + parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") + args = parser.parse_args() + + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.device = device + + if args.tpu: + if args.tpu_ip_address: + os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address + if args.tpu_name: + os.environ["TPU_NAME"] = args.tpu_name + if args.xrt_tpu_config: + os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config + + assert "TPU_IP_ADDRESS" in os.environ + assert "TPU_NAME" in os.environ + assert "XRT_TPU_CONFIG" in os.environ + + import torch_xla + import torch_xla.core.xla_model as xm + args.device = xm.xla_device() + args.xla_model = xm + + # Setup logging + logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + + # Set seed + set_seed(args) + + # Prepare XNLI task + args.task_name = 'xnli' + if args.task_name not in processors: + raise ValueError("Task not found: %s" % (args.task_name)) + processor = processors[args.task_name](language=args.language, train_language=args.train_language) + args.output_mode = output_modes[args.task_name] + label_list = processor.get_labels() + num_labels = len(label_list) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) + model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + model.to(args.device) + + logger.info("Training/evaluation parameters %s", args) + + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False) + global_step, tr_loss = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + + # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir) + tokenizer.save_pretrained(args.output_dir) + + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + # Load a trained model and vocabulary that you have fine-tuned + model = model_class.from_pretrained(args.output_dir) + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + model.to(args.device) + + + # Evaluation + results = {} + if args.do_eval and args.local_rank in [-1, 0]: + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logger.info("Evaluate the following checkpoints: %s", checkpoints) + for checkpoint in checkpoints: + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else "" + + model = model_class.from_pretrained(checkpoint) + model.to(args.device) + result = evaluate(args, model, tokenizer, prefix=prefix) + result = dict((k + '_{}'.format(global_step), v) for k, v in result.items()) + results.update(result) + + return results + + +if __name__ == "__main__": + main() diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py new file mode 100644 index 0000000000..f0238f4664 --- /dev/null +++ b/examples/utils_xnli.py @@ -0,0 +1,93 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XNLI utils (dataset loading and evaluation) """ + +from __future__ import absolute_import, division, print_function + +import logging +import os + +from transformers.data.processors import DataProcessor, InputExample +from transformers.data.metrics import simple_accuracy + +logger = logging.getLogger(__name__) + +class XnliProcessor(DataProcessor): + """Processor for the XNLI dataset. + Adapted from https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/run_classifier.py#L207""" + + def __init__(self, language, train_language = None): + self.language = language + self.train_language = train_language + + def get_train_examples(self, data_dir): + """See base class.""" + lg = self.language if self.train_language is None else self.train_language + lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % ('train', i) + text_a = line[0] + text_b = line[1] + label = "contradiction" if line[2] == "contradictory" else line[2] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + language = line[0] + if language != self.language: + continue + guid = "%s-%s" % ('dev', i) + text_a = line[6] + text_b = line[7] + label = line[1] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + +def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise ValueError(f'{task_name} is not a supported task.') + +xnli_processors = { + "xnli": XnliProcessor, +} + +xnli_output_modes = { + "xnli": "classification", +} + +xnli_tasks_num_labels = { + "xnli": 3, +} From d52e98ff9af4509bb803641f0c0d81d67ce73cc3 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 11:51:15 -0400 Subject: [PATCH 129/143] add xnli examples/README.md --- examples/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/examples/README.md b/examples/README.md index e109a12171..6f8d6bd26e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -21,6 +21,7 @@ pip install [--editable] . | [SQuAD](#squad) | Using BERT/RoBERTa/XLNet/XLM for question answering, examples with distributed training. | | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training. | +| [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. | | [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. | ## TensorFlow 2.0 Bert models on GLUE @@ -600,3 +601,42 @@ python run_summarization_finetuning.py \ --do_train \ --data_path=$DATA_PATH \ ``` + +## XNLI + +Based on the script [`run_xnli.py`](TODO). + +[XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +#### Fine-tuning on XNLI + +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min +on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a +`$XNLI_DIR` directory. + +* [XNLI 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip) +* [XNLI-MT 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-MT-1.0.zip) + +```bash +export XNLI_DIR=/path/to/XNLI + +python run_xnli.py \ + --model_type bert \ + --model_name_or_path bert-base-multilingual-cased \ + --language en \ + --train_language en \ + --do_train \ + --do_eval \ + --data_dir $SQUAD_DIR \ + --per_gpu_train_batch_size 32 \ + --learning_rate 5e-5 \ + --num_train_epochs 2.0 \ + --max_seq_length 128 \ + --output_dir /tmp/debug_xnli/ +``` + +Training with the previously defined hyper-parameters yields the following results: + +```bash +TODO +``` From c4336ecbbdbb8bddcbfb31e0fcc4d382b430a9a5 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 12:04:20 -0400 Subject: [PATCH 130/143] xnli - output_mode consistency --- examples/run_xnli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index ee37296832..a9b2e46c13 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -247,8 +247,8 @@ def evaluate(args, model, tokenizer, prefix=""): eval_loss = eval_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) - elif args.output_mode == "regression": - preds = np.squeeze(preds) + else: + raise ValueError(f'No other `output_mode` for XNLI.') result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) From 84a0b522cf1b3f69bfbc92eb2309ba5ff652e521 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 18:53:45 -0400 Subject: [PATCH 131/143] mbert reproducibility results --- examples/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/README.md b/examples/README.md index 6f8d6bd26e..3e5fd03c45 100644 --- a/examples/README.md +++ b/examples/README.md @@ -604,13 +604,13 @@ python run_summarization_finetuning.py \ ## XNLI -Based on the script [`run_xnli.py`](TODO). +Based on the script [`run_xnli.py`](https://github.com/huggingface/transformers/blob/master/examples/run_xnli.py). [XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). #### Fine-tuning on XNLI -This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in 106 mins on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a `$XNLI_DIR` directory. @@ -623,20 +623,21 @@ export XNLI_DIR=/path/to/XNLI python run_xnli.py \ --model_type bert \ --model_name_or_path bert-base-multilingual-cased \ - --language en \ + --language es \ --train_language en \ --do_train \ --do_eval \ - --data_dir $SQUAD_DIR \ + --data_dir $XNLI_DIR \ --per_gpu_train_batch_size 32 \ --learning_rate 5e-5 \ --num_train_epochs 2.0 \ --max_seq_length 128 \ - --output_dir /tmp/debug_xnli/ + --output_dir /tmp/debug_xnli/ \ + --save_steps -1 ``` -Training with the previously defined hyper-parameters yields the following results: +Training with the previously defined hyper-parameters yields the following results on the dev set: ```bash -TODO +acc = 0.738152610441767 ``` From cb7b77a8a2ee52812c4358817a6a586f19687cda Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 30 Oct 2019 18:13:52 -0400 Subject: [PATCH 132/143] fix some typos --- transformers/tokenization_xlm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index 01f8721d98..ba994dc356 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tokenization classes for OpenAI GPT.""" +"""Tokenization classes for XLM.""" from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -758,9 +758,9 @@ class XLMTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: + A XLM sequence has the following format: single sequence: X - pair of sequences: A B + pair of sequences: A B """ if token_ids_1 is None: return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] From 289cf4d2b7cf090942e2e7dc4e6134a81095adf6 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:10:02 -0500 Subject: [PATCH 133/143] change default for XNLI: dev --> test --- examples/run_xnli.py | 6 +++--- examples/utils_xnli.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index a9b2e46c13..7fbbf7d152 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -270,7 +270,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): output_mode = output_modes[task] # Load data features from cache or dataset file cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( - 'dev' if evaluate else 'train', + 'test' if evaluate else 'train', list(filter(None, args.model_name_or_path.split('/'))).pop(), str(args.max_seq_length), str(task), @@ -281,7 +281,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): else: logger.info("Creating features from dataset file at %s", args.data_dir) label_list = processor.get_labels() - examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + examples = processor.get_test_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, tokenizer, label_list=label_list, @@ -341,7 +341,7 @@ def main(): parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', - help="Whether to run eval on the dev set.") + help="Whether to run eval on the test set.") parser.add_argument("--evaluate_during_training", action='store_true', help="Rul evaluation during training at each logging step.") parser.add_argument("--do_lower_case", action='store_true', diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py index f0238f4664..482e79a81f 100644 --- a/examples/utils_xnli.py +++ b/examples/utils_xnli.py @@ -50,9 +50,9 @@ class XnliProcessor(DataProcessor): InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples - def get_dev_examples(self, data_dir): + def get_test_examples(self, data_dir): """See base class.""" - lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv")) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv")) examples = [] for (i, line) in enumerate(lines): if i == 0: @@ -60,7 +60,7 @@ class XnliProcessor(DataProcessor): language = line[0] if language != self.language: continue - guid = "%s-%s" % ('dev', i) + guid = "%s-%s" % ('test', i) text_a = line[6] text_b = line[7] label = line[1] From d5910b312fbe02a4b08097646e46399dac179084 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:21:25 -0500 Subject: [PATCH 134/143] move xnli processor (and utils) to transformers/data/processors --- examples/run_xnli.py | 6 +++--- transformers/__init__.py | 3 ++- transformers/data/__init__.py | 1 + transformers/data/processors/__init__.py | 2 +- .../utils_xnli.py => transformers/data/processors/xnli.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) rename examples/utils_xnli.py => transformers/data/processors/xnli.py (97%) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 7fbbf7d152..0c1743e6c4 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -44,9 +44,9 @@ from transformers import (WEIGHTS_NAME, from transformers import AdamW, WarmupLinearSchedule -from utils_xnli import xnli_compute_metrics as compute_metrics -from utils_xnli import xnli_output_modes as output_modes -from utils_xnli import xnli_processors as processors +from transformers import xnli_compute_metrics as compute_metrics +from transformers import xnli_output_modes as output_modes +from transformers import xnli_processors as processors from transformers import glue_convert_examples_to_features as convert_examples_to_features diff --git a/transformers/__init__.py b/transformers/__init__.py index dff2479d4b..a133425a9c 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -25,7 +25,8 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, glue_output_modes, glue_convert_examples_to_features, - glue_processors, glue_tasks_num_labels) + glue_processors, glue_tasks_num_labels, + xnli_output_modes, xnli_processors, xnli_tasks_num_labels) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index e910d6da2e..46615608a4 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,5 +1,6 @@ from .processors import InputExample, InputFeatures, DataProcessor from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_labels from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index af38c54beb..1c41553ba4 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,3 +1,3 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features - +from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels diff --git a/examples/utils_xnli.py b/transformers/data/processors/xnli.py similarity index 97% rename from examples/utils_xnli.py rename to transformers/data/processors/xnli.py index 482e79a81f..96fac761a9 100644 --- a/examples/utils_xnli.py +++ b/transformers/data/processors/xnli.py @@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function import logging import os -from transformers.data.processors import DataProcessor, InputExample +from .utils import DataProcessor, InputExample from transformers.data.metrics import simple_accuracy logger = logging.getLogger(__name__) From d75d49a51de28df48b39280b2f06df78d3fa04bf Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:32:20 -0500 Subject: [PATCH 135/143] add XnliProcessor to doc --- docs/source/main_classes/processors.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index a85c126956..a093e621ad 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -55,4 +55,27 @@ Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ An example using these processors is given in the -`run_glue.py `__ script. \ No newline at end of file +`run_glue.py `__ script. + + +XNLI +~~~~~~~~~~~~~~~~~~~~~ + +`The Cross-Lingual NLI Corpus (XNLI) `__ is a benchmark that evaluates +the quality of cross-lingual text representations. +XNLI is crowd-sourced dataset based on `MultiNLI `: pairs of text are labeled with textual entailment +annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +It was released together with the paper +`XNLI: Evaluating Cross-lingual Sentence Representations `__ + +This library hosts the processor to load the XNLI data: + - :class:`~transformers.data.processors.utils.XnliProcessor` + +Please note that since the gold labels are available on the test set, evaluation is performed on the test set. + +Example usage +^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example using these processors is given in the +`run_xnli.py `__ script. \ No newline at end of file From abd397e95483cc7486bed47fd271ab417e1a288e Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:57:24 -0500 Subject: [PATCH 136/143] uniformize w/ the cache_dir update --- examples/run_xnli.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 0c1743e6c4..e8457e542b 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -467,9 +467,17 @@ def main(): args.model_type = args.model_type.lower() config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] - config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) - tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) - model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, + num_labels=num_labels, + finetuning_task=args.task_name, + cache_dir=args.cache_dir if args.cache_dir else None) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, + do_lower_case=args.do_lower_case, + cache_dir=args.cache_dir if args.cache_dir else None) + model = model_class.from_pretrained(args.model_name_or_path, + from_tf=bool('.ckpt' in args.model_name_or_path), + config=config, + cache_dir=args.cache_dir if args.cache_dir else None) if args.local_rank == 0: torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab From 3e7656f7ac369de08a2ebac3cd1d6870e60605bc Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 11:58:53 -0500 Subject: [PATCH 137/143] update readme --- examples/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/README.md b/examples/README.md index 3e5fd03c45..960b218f11 100644 --- a/examples/README.md +++ b/examples/README.md @@ -623,7 +623,7 @@ export XNLI_DIR=/path/to/XNLI python run_xnli.py \ --model_type bert \ --model_name_or_path bert-base-multilingual-cased \ - --language es \ + --language de \ --train_language en \ --do_train \ --do_eval \ @@ -636,8 +636,8 @@ python run_xnli.py \ --save_steps -1 ``` -Training with the previously defined hyper-parameters yields the following results on the dev set: +Training with the previously defined hyper-parameters yields the following results on the **test** set: ```bash -acc = 0.738152610441767 +acc = 0.7093812375249501 ``` From 73fe2e7385af4c6062f366825af570d44cd22fd8 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:51:43 -0500 Subject: [PATCH 138/143] remove fstrings --- examples/run_xnli.py | 4 ++-- transformers/data/processors/xnli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index e8457e542b..952128f4ad 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -248,7 +248,7 @@ def evaluate(args, model, tokenizer, prefix=""): if args.output_mode == "classification": preds = np.argmax(preds, axis=1) else: - raise ValueError(f'No other `output_mode` for XNLI.') + raise ValueError('No other `output_mode` for XNLI.') result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) @@ -305,7 +305,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): if output_mode == "classification": all_labels = torch.tensor([f.label for f in features], dtype=torch.long) else: - raise ValueError(f'No other `output_mode` for XNLI.') + raise ValueError('No other `output_mode` for XNLI.') dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index 96fac761a9..a4807dd901 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -36,7 +36,7 @@ class XnliProcessor(DataProcessor): def get_train_examples(self, data_dir): """See base class.""" lg = self.language if self.train_language is None else self.train_language - lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg))) examples = [] for (i, line) in enumerate(lines): if i == 0: @@ -78,7 +78,7 @@ def xnli_compute_metrics(task_name, preds, labels): if task_name == "xnli": return {"acc": simple_accuracy(preds, labels)} else: - raise ValueError(f'{task_name} is not a supported task.') + raise ValueError('{} is not a supported task.'.format(task_name)) xnli_processors = { "xnli": XnliProcessor, From bcd8dc6b48a335b899f00b59f016f740c5230d41 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:53:08 -0500 Subject: [PATCH 139/143] move xnli_compute_metrics to data/metrics --- transformers/__init__.py | 2 +- transformers/data/__init__.py | 2 +- transformers/data/metrics/__init__.py | 8 ++++++++ transformers/data/processors/xnli.py | 7 ------- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index a133425a9c..b29ad38e73 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -29,7 +29,7 @@ from .data import (is_sklearn_available, xnli_output_modes, xnli_processors, xnli_tasks_num_labels) if is_sklearn_available(): - from .data import glue_compute_metrics + from .data import glue_compute_metrics, xnli_compute_metrics # Tokenizers from .tokenization_utils import (PreTrainedTokenizer) diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index 46615608a4..b811a35807 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -4,4 +4,4 @@ from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_label from .metrics import is_sklearn_available if is_sklearn_available(): - from .metrics import glue_compute_metrics + from .metrics import glue_compute_metrics, xnli_compute_metrics diff --git a/transformers/data/metrics/__init__.py b/transformers/data/metrics/__init__.py index c9ebaac38d..5a46eb05d3 100644 --- a/transformers/data/metrics/__init__.py +++ b/transformers/data/metrics/__init__.py @@ -81,3 +81,11 @@ if _has_sklearn: return {"acc": simple_accuracy(preds, labels)} else: raise KeyError(task_name) + + + def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index a4807dd901..ce582f31a6 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -73,13 +73,6 @@ class XnliProcessor(DataProcessor): """See base class.""" return ["contradiction", "entailment", "neutral"] -def xnli_compute_metrics(task_name, preds, labels): - assert len(preds) == len(labels) - if task_name == "xnli": - return {"acc": simple_accuracy(preds, labels)} - else: - raise ValueError('{} is not a supported task.'.format(task_name)) - xnli_processors = { "xnli": XnliProcessor, } From d47402263964e30ee17cbc06811622bf2df50d6d Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:56:03 -0500 Subject: [PATCH 140/143] cleaning simple_accuracy since not used anymore --- transformers/data/processors/xnli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index ce582f31a6..efbe3762f6 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -21,7 +21,6 @@ import logging import os from .utils import DataProcessor, InputExample -from transformers.data.metrics import simple_accuracy logger = logging.getLogger(__name__) From 07ab8d7af6041f0d3562badcb3d6f173062329a1 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 17:33:14 -0500 Subject: [PATCH 141/143] fix bug --- transformers/data/processors/xnli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index efbe3762f6..958bdf62f9 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -35,7 +35,7 @@ class XnliProcessor(DataProcessor): def get_train_examples(self, data_dir): """See base class.""" lg = self.language if self.train_language is None else self.train_language - lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg))) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{}.tsv".format(lg))) examples = [] for (i, line) in enumerate(lines): if i == 0: From d5478b939d64db58972e46b7218c765c918b76ac Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 25 Nov 2019 19:40:48 +0000 Subject: [PATCH 142/143] add distilbert + update run_xnli wrt run_glue --- examples/run_xnli.py | 51 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 952128f4ad..a3bc0d4604 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Finetuning multi-lingual models on XNLI (Bert, XLM). +""" Finetuning multi-lingual models on XNLI (Bert, DistilBERT, XLM). Adapted from `examples/run_glue.py`""" from __future__ import absolute_import, division, print_function @@ -42,7 +42,7 @@ from transformers import (WEIGHTS_NAME, XLMConfig, XLMForSequenceClassification, XLMTokenizer, DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from transformers import xnli_compute_metrics as compute_metrics from transformers import xnli_output_modes as output_modes @@ -52,12 +52,12 @@ from transformers import glue_convert_examples_to_features as convert_examples_t logger = logging.getLogger(__name__) -ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ()) +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, DistilBertConfig, XLMConfig)), ()) MODEL_CLASSES = { 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), - # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) } @@ -91,7 +91,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp @@ -149,7 +149,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -180,11 +180,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -214,6 +209,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu eval + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) @@ -383,15 +382,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -425,23 +415,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -495,7 +468,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) @@ -512,7 +485,7 @@ def main(): # Load a trained model and vocabulary that you have fine-tuned model = model_class.from_pretrained(args.output_dir) - tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + tokenizer = tokenizer_class.from_pretrained(args.output_dir) model.to(args.device) From 10bd1ddb39235b2f58594e48867595e7d38cd619 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 25 Nov 2019 19:41:00 +0000 Subject: [PATCH 143/143] soft launch distilbert multilingual --- transformers/configuration_distilbert.py | 3 ++- transformers/modeling_distilbert.py | 3 ++- transformers/tokenization_distilbert.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index 2a8a149acf..f929a9bc39 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -27,7 +27,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-config.json", } diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index d30f493c69..62c623ff6c 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -42,7 +42,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-pytorch_model.bin", } diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index dfa02926d8..832f0c3d0b 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -33,12 +33,14 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'distilbert-base-uncased': 512, 'distilbert-base-uncased-distilled-squad': 512, + 'distilbert-base-multilingual-cased': 512, }