From 0e4cc050d63058969c53cce295dea99e90483c01 Mon Sep 17 00:00:00 2001 From: Sergey Mironov Date: Thu, 24 Oct 2019 18:15:55 +0300 Subject: [PATCH 01/91] Add support for resumable downloads for HTTP protocol. --- transformers/configuration_auto.py | 3 ++ transformers/configuration_utils.py | 7 ++++- transformers/file_utils.py | 47 ++++++++++++++++++++++------- transformers/modeling_auto.py | 8 +++++ transformers/modeling_tf_auto.py | 12 ++++++++ transformers/modeling_tf_utils.py | 8 ++++- transformers/modeling_utils.py | 8 ++++- transformers/tokenization_auto.py | 3 ++ transformers/tokenization_utils.py | 6 +++- 9 files changed, 87 insertions(+), 15 deletions(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a670c..0e8eb03d03 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -92,6 +92,9 @@ class AutoConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index cfa6502bcd..0f9d609b20 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -93,6 +93,9 @@ class PretrainedConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -119,6 +122,7 @@ class PretrainedConfig(object): """ cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) @@ -130,7 +134,8 @@ class PretrainedConfig(object): config_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: - resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_config_archive_map: msg = "Couldn't reach server at '{}' to download pretrained model configuration file.".format( diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 27875212ff..24abd60781 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -22,6 +22,7 @@ from botocore.config import Config from botocore.exceptions import ClientError import requests from tqdm import tqdm +from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -152,7 +153,7 @@ def filename_to_url(filename, cache_dir=None): return url, etag -def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None): +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False): """ Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and @@ -161,6 +162,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N Args: cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). force_download: if True, re-dowload the file even if it's already cached in the cache dir. + resume_download: if True, resume the download if incompletly recieved file is found. """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -173,7 +175,9 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N if parsed.scheme in ('http', 'https', 's3'): # URL, so get it from the cache (downloading if necessary) - return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + return get_from_cache(url_or_filename, cache_dir=cache_dir, + force_download=force_download, proxies=proxies, + resume_download=resume_download) elif os.path.exists(url_or_filename): # File, and it exists. return url_or_filename @@ -234,19 +238,22 @@ def s3_get(url, temp_file, proxies=None): s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) -def http_get(url, temp_file, proxies=None): - req = requests.get(url, stream=True, proxies=proxies) - content_length = req.headers.get('Content-Length') - total = int(content_length) if content_length is not None else None - progress = tqdm(unit="B", total=total) - for chunk in req.iter_content(chunk_size=1024): +def http_get(url, temp_file, proxies=None, resume_size=0): + headers={'Range':'bytes=%d-'%(resume_size,)} if resume_size > 0 else None + response = requests.get(url, stream=True, proxies=proxies, headers=headers) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get('Content-Length') + total = resume_size + int(content_length) if content_length is not None else None + progress = tqdm(unit="B", total=total, initial=resume_size) + for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) temp_file.write(chunk) progress.close() -def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10): +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False): """ Given a URL, look for the corresponding dataset in the local cache. If it's not there, download it. Then return the path to the cached file. @@ -289,17 +296,35 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag if matching_files: cache_path = os.path.join(cache_dir, matching_files[-1]) + if resume_download: + incomplete_path = cache_path + '.incomplete' + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path,'a+b') as f: + yield f + os.remove(incomplete_path) + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = tempfile.NamedTemporaryFile + resume_size = 0 + if not os.path.exists(cache_path) or force_download: # Download to temporary file, then copy to cache dir once finished. # Otherwise you get corrupt cache entries if the download gets interrupted. - with tempfile.NamedTemporaryFile() as temp_file: + with temp_file_manager() as temp_file: logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) # GET file object if url.startswith("s3://"): + if resume_download: + logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') s3_get(url, temp_file, proxies=proxies) else: - http_get(url, temp_file, proxies=proxies) + http_get(url, temp_file, proxies=proxies, resume_size=resume_size) # we are copying the file before closing it, so flush to avoid truncation temp_file.flush() diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..ea5a539c8c 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -112,6 +112,9 @@ class AutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -237,6 +240,8 @@ class AutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. @@ -357,6 +362,9 @@ class AutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index df0ad6e401..cfe19ead2a 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -109,6 +109,9 @@ class TFAutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -237,6 +240,9 @@ class TFAutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -360,6 +366,9 @@ class TFAutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -472,6 +481,9 @@ class TFAutoModelForQuestionAnswering(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index a96e2765fd..aed2a25643 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -176,6 +176,9 @@ class TFPreTrainedModel(tf.keras.Model): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -201,6 +204,7 @@ class TFPreTrainedModel(tf.keras.Model): cache_dir = kwargs.pop('cache_dir', None) from_pt = kwargs.pop('from_pt', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) # Load config @@ -209,6 +213,7 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -236,7 +241,8 @@ class TFPreTrainedModel(tf.keras.Model): # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + resume_download=resume_download, proxies=proxies) except EnvironmentError as e: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: logger.error( diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d082137d5d..db3ab93f8f 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -246,6 +246,9 @@ class PreTrainedModel(nn.Module): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -275,6 +278,7 @@ class PreTrainedModel(nn.Module): cache_dir = kwargs.pop('cache_dir', None) from_tf = kwargs.pop('from_tf', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) output_loading_info = kwargs.pop('output_loading_info', False) @@ -284,6 +288,7 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -315,7 +320,8 @@ class PreTrainedModel(nn.Module): # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: msg = "Couldn't reach server at '{}' to download pretrained weights.".format( diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de17f..455f503a9d 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -87,6 +87,9 @@ class AutoTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 5e5be872ef..b602ebf20a 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -251,6 +251,9 @@ class PreTrainedTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -286,6 +289,7 @@ class PreTrainedTokenizer(object): def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) s3_models = list(cls.max_model_input_sizes.keys()) @@ -352,7 +356,7 @@ class PreTrainedTokenizer(object): if file_path is None: resolved_vocab_files[file_id] = None else: - resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in s3_models: msg = "Couldn't reach server at '{}' to download vocabulary files." From 8d6b9d717c676560d291a7d2913b44e1cf226cd5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 4 Nov 2019 17:07:51 +0100 Subject: [PATCH 02/91] fix #1532 and encode_plus --- .../tests/tokenization_tests_commons.py | 21 ++++-- transformers/tokenization_utils.py | 74 ++++++++++++++++--- 2 files changed, 77 insertions(+), 18 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..cb137e8395 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -223,7 +223,11 @@ class CommonTestCases: sequence = tokenizer.encode(seq_0, add_special_tokens=False) num_added_tokens = tokenizer.num_added_tokens() total_length = len(sequence) + num_added_tokens - information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) + information = tokenizer.encode_plus(seq_0, + max_length=total_length - 2, + add_special_tokens=True, + stride=stride, + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -250,10 +254,12 @@ class CommonTestCases: ) information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, - stride=stride, truncation_strategy='only_second') + stride=stride, truncation_strategy='only_second', + return_overflowing_tokens=True) information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, stride=stride, - truncation_strategy='only_first') + truncation_strategy='only_first', + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -285,7 +291,7 @@ class CommonTestCases: # Testing single inputs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True, return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -297,7 +303,8 @@ class CommonTestCases: # Testing inputs pairs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) + tokenizer.encode(sequence_1, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -309,7 +316,9 @@ class CommonTestCases: # Testing with already existing special tokens if tokenizer.cls_token_id == tokenizer.unk_token_id and tokenizer.cls_token_id == tokenizer.unk_token_id: tokenizer.add_special_tokens({'cls_token': '', 'sep_token': ''}) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, + add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask_orig = encoded_sequence_dict["special_tokens_mask"] special_tokens_mask = tokenizer.get_special_tokens_mask(encoded_sequence_w_special, already_has_special_tokens=True) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index ac765165e2..f860755775 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -744,6 +744,9 @@ class PreTrainedTokenizer(object): stride=0, truncation_strategy='longest_first', return_tensors=None, + return_token_type_ids=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair and additional informations: @@ -770,7 +773,30 @@ class PreTrainedTokenizer(object): - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). **kwargs: passed to the `self.tokenize()` method + + Return: + A Dictionary of shape:: + + { + input_ids: list[int], + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True + } + + With the fields: + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model + + ``overflowing_tokens``: list of overflowing tokens if a max length is specified. + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified + ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added + tokens and 1 specifying sequence tokens. """ def get_input_ids(text): @@ -792,10 +818,17 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - return_tensors=return_tensors) + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask) def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, - truncation_strategy='longest_first', return_tensors=None): + truncation_strategy='longest_first', + return_tensors=None, + return_token_type_ids=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False): """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates @@ -820,21 +853,27 @@ class PreTrainedTokenizer(object): - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). Return: A Dictionary of shape:: { input_ids: list[int], - overflowing_tokens: list[int] if a ``max_length`` is specified, else None - special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True } With the fields: - ``input_ids``: list of tokens to be fed to a model + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model ``overflowing_tokens``: list of overflowing tokens if a max length is specified. - + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added tokens and 1 specifying sequence tokens. """ @@ -843,23 +882,31 @@ class PreTrainedTokenizer(object): len_pair_ids = len(pair_ids) if pair else 0 encoded_inputs = {} + + # Handle max sequence length total_len = len_ids + len_pair_ids + (self.num_added_tokens(pair=pair) if add_special_tokens else 0) if max_length and total_len > max_length: ids, pair_ids, overflowing_tokens = self.truncate_sequences(ids, pair_ids=pair_ids, num_tokens_to_remove=total_len-max_length, truncation_strategy=truncation_strategy, stride=stride) - encoded_inputs["overflowing_tokens"] = overflowing_tokens - encoded_inputs["num_truncated_tokens"] = total_len - max_length + if return_overflowing_tokens: + encoded_inputs["overflowing_tokens"] = overflowing_tokens + encoded_inputs["num_truncated_tokens"] = total_len - max_length + # Handle special_tokens if add_special_tokens: sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) - encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + special_tokens_mask = self.get_special_tokens_mask(ids, pair_ids) else: sequence = ids + pair_ids if pair else ids token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) + special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0)) + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + # Prepare inputs as tensors if asked if return_tensors == 'tf' and is_tf_available(): sequence = tf.constant([sequence]) token_type_ids = tf.constant([token_type_ids]) @@ -870,12 +917,15 @@ class PreTrainedTokenizer(object): logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) encoded_inputs["input_ids"] = sequence - encoded_inputs["token_type_ids"] = token_type_ids + if return_token_type_ids: + encoded_inputs["token_type_ids"] = token_type_ids if max_length and len(encoded_inputs["input_ids"]) > max_length: encoded_inputs["input_ids"] = encoded_inputs["input_ids"][:max_length] - encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] - encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] + if return_token_type_ids: + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] return encoded_inputs From 7246d3c2f93c4461f3ec8ada7a26a002d8f196ea Mon Sep 17 00:00:00 2001 From: Michael Watkins Date: Wed, 6 Nov 2019 13:18:16 +0200 Subject: [PATCH 03/91] Consider do_lower_case in PreTrainedTokenizer As pointed out in #1545, when using an uncased model, and adding a new uncased token, the tokenizer does not correctly identify this in the case that the input text contains the token in a cased format. For instance, if we load bert-base-uncased into BertTokenizer, and then use .add_tokens() to add "cool-token", we get the expected result for .tokenize('this is a cool-token'). However, we get a possibly unexpected result for .tokenize('this is a cOOl-Token'), which in fact mirrors the result for the former from before the new token was added. This commit adds - functionality to PreTrainedTokenizer to handle this situation in case a tokenizer (currently Bert, DistilBert, and XLNet) has the do_lower_case=True kwarg by: 1) lowercasing tokens added with .add_tokens() 2) lowercasing text at the beginning of .tokenize() - new common test case for tokenizers https://github.com/huggingface/transformers/issues/1545 --- .../tests/tokenization_tests_commons.py | 31 ++++++++++++++++++- transformers/tokenization_utils.py | 5 +++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..287e6fc7b3 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -110,6 +110,36 @@ class CommonTestCases: self.assertListEqual(subwords, subwords_loaded) + def test_added_tokens_do_lower_case(self): + tokenizer = self.get_tokenizer(do_lower_case=True) + + text = "aaaaa bbbbbb low cccccccccdddddddd l" + text2 = "AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l" + + toks0 = tokenizer.tokenize(text) # toks before adding new_toks + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", 'AAAAA BBBBBB', 'CCCCCCCCCDDDDDDDD'] + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 2) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) + self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer + self.assertListEqual(toks, toks2) + + tokenizer = self.get_tokenizer(do_lower_case=False) + + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 4) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) # Length should still be the same + self.assertNotEqual(len(toks), len(toks0)) + self.assertNotEqual(toks[0], toks2[0]) # But at least the first tokens should differ def test_add_tokens_tokenizer(self): tokenizer = self.get_tokenizer() @@ -160,7 +190,6 @@ class CommonTestCases: self.assertEqual(tokens[0], tokenizer.eos_token_id) self.assertEqual(tokens[-2], tokenizer.pad_token_id) - def test_required_methods_tokenizer(self): tokenizer = self.get_tokenizer() input_text, output_text = self.get_input_output_texts() diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cd14cc4582..fc31c10d25 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -512,6 +512,8 @@ class PreTrainedTokenizer(object): to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) + if self.init_kwargs.get('do_lower_case', False): + token = token.lower() if token != self.unk_token and \ self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ token not in to_add_tokens: @@ -605,6 +607,9 @@ class PreTrainedTokenizer(object): Take care of added tokens. """ + if self.init_kwargs.get('do_lower_case', False): + text = text.lower() + def split_on_token(tok, text): result = [] split_text = text.split(tok) From d32ce2c8df7053c19061b709465cdcc765e45a15 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 14:14:19 +0100 Subject: [PATCH 04/91] camembert: add wrapper for CamembertForTokenClassification --- transformers/modeling_camembert.py | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index 05538926e2..f302346f2d 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function, import logging -from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification from .configuration_camembert import CamembertConfig from .file_utils import add_start_docstrings @@ -255,3 +255,39 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForTokenClassification(RobertaForTokenClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForTokenClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !", add_special_tokens=True)).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP From 33753d9139307d9635db0309b6ddb9c53192c60a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 14:14:54 +0100 Subject: [PATCH 05/91] module: import CamembertForTokenClassification --- transformers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index cdf0669b39..5c7b0a6197 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -100,6 +100,7 @@ if is_torch_available(): DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_camembert import (CamembertForMaskedLM, CamembertModel, CamembertForSequenceClassification, CamembertForMultipleChoice, + CamembertForTokenClassification, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model From 0b3d45eb64607158977f546d57f90eae268c7836 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 15:49:44 +0100 Subject: [PATCH 06/91] camembert: add implementation for save_vocabulary method --- transformers/tokenization_camembert.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 41d3d74cff..bf2a6fe993 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -16,9 +16,14 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import logging +import os +from shutil import copyfile + import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +logger = logging.getLogger(__name__) VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} @@ -55,6 +60,7 @@ class CamembertTokenizer(PreTrainedTokenizer): self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(str(vocab_file)) + self.vocab_file = vocab_file # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual # sentencepiece vocabulary (this is the case for and self.fairseq_tokens_to_ids = {'NOTUSED': 0, '': 1, 'NOTUSED': 2, '': 3} @@ -135,3 +141,17 @@ class CamembertTokenizer(PreTrainedTokenizer): if index in self.fairseq_ids_to_tokens: return self.fairseq_ids_to_tokens[index] return self.sp_model.IdToPiece(index - self.fairseq_offset) + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From 56c84863a1a20dfb82b928c5c9f77c21d9def8c7 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 15:50:16 +0100 Subject: [PATCH 07/91] camembert: add support for CamemBERT in run_ner example --- examples/run_ner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 4359e587ae..127d63a6cd 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -37,6 +37,7 @@ from transformers import AdamW, get_linear_schedule_with_warmup from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer +from transformers import CamembertConfig, CamembertForTokenClassification, CamembertTokenizer logger = logging.getLogger(__name__) @@ -47,7 +48,8 @@ ALL_MODELS = sum( MODEL_CLASSES = { "bert": (BertConfig, BertForTokenClassification, BertTokenizer), "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer), - "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer) + "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer), + "camembert": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer), } From 454455c695ff38df1ed3670a43677fdd1abcedf3 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 20 Nov 2019 09:42:48 -0500 Subject: [PATCH 08/91] fix #1879 --- examples/utils_squad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/utils_squad.py b/examples/utils_squad.py index c04dacf6d3..4f1c581588 100644 --- a/examples/utils_squad.py +++ b/examples/utils_squad.py @@ -240,6 +240,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length, # The -3 accounts for [CLS], [SEP] and [SEP] max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + assert max_tokens_for_doc > 0 # We can have documents that are longer than the maximum sequence length. # To deal with this we do a sliding window approach, where we take chunks From e70cdf083ddb8bfe298d43e6d70d698a3a2f56d3 Mon Sep 17 00:00:00 2001 From: Jin Young Sohn Date: Wed, 20 Nov 2019 22:33:26 +0000 Subject: [PATCH 09/91] Cleanup TPU bits from run_glue.py TPU runner is currently implemented in: https://github.com/pytorch-tpu/transformers/blob/tpu/examples/run_glue_tpu.py. We plan to upstream this directly into `huggingface/transformers` (either `master` or `tpu`) branch once it's been more thoroughly tested. --- examples/run_glue.py | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 19316cb0ec..527e440075 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -158,7 +158,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -189,11 +189,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -397,15 +392,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -439,23 +425,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -509,7 +478,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) From 6f70bb8c69cdbe1207e4909ccface32c8f51297b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 20 Nov 2019 18:01:03 +0100 Subject: [PATCH 10/91] add instructions to run the examples --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index a49e8086d1..9cc783fd71 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,18 @@ When TensorFlow 2.0 and/or PyTorch has been installed, you can install from sour pip install [--editable] . ``` +### Run the examples + +Examples are included in the repository but are not shipped with the library. +Therefore, in order to run the examples you will first need to clone the +repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps: + +```bash +git clone git@github.com:huggingface/transformers +cd transformers +pip install . +``` + ### Tests A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). @@ -253,6 +265,11 @@ print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sen ## Quick tour of the fine-tuning/usage scripts +**Important** +Before running the fine-tuning scripts, please read the +[instructions](#run-the-examples) on how to +setup your environment to run the examples. + The library comprises several example scripts with SOTA performances for NLU and NLG tasks: - `run_glue.py`: an example fine-tuning Bert, XLNet and XLM on nine different GLUE tasks (*sequence-level classification*) From 26db31e0c09a8b5e1ca7a61c454b159eab9d86be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 20 Nov 2019 18:13:38 +0100 Subject: [PATCH 11/91] update the documentation --- examples/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/README.md b/examples/README.md index abb4cb6e5a..622fa07f8f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,6 +3,15 @@ In this section a few examples are put together. All of these examples work for several models, making use of the very similar API between the different models. +**Important** +To use the examples, execute the following steps in a new virtual environment: + +```bash +git clone git@github.com:huggingface/transformers +cd transformers +pip install . +``` + | Section | Description | |----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| | [TensorFlow 2.0 models on GLUE](#TensorFlow-2.0-Bert-models-on-GLUE) | Examples running BERT TensorFlow 2.0 model on the GLUE tasks. From 041a901f324eea7e7ee04b0f7a563c7ed5c8a03a Mon Sep 17 00:00:00 2001 From: Nikolay Korolev Date: Fri, 22 Nov 2019 22:56:43 +0300 Subject: [PATCH 12/91] Fix typo in documentation. toto -> to --- transformers/tokenization_gpt2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 4bec515903..a4798b0129 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -184,7 +184,7 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ Tokenize a string. Args: - add_prefix_space (boolean, default False): - Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers. + Begin the sentence with at least one space to get invariance to word order in GPT-2 (and RoBERTa) tokenizers. """ if add_prefix_space: text = ' ' + text From 176cd1ce1b337134425b426207fbe155099c18b4 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 23 Nov 2019 11:18:54 -0500 Subject: [PATCH 13/91] [doc] homogenize instructions slightly --- README.md | 5 ++--- examples/README.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9cc783fd71..52e74a6f80 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,12 @@ pip install [--editable] . ### Run the examples Examples are included in the repository but are not shipped with the library. -Therefore, in order to run the examples you will first need to clone the -repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps: +Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: ```bash git clone git@github.com:huggingface/transformers cd transformers -pip install . +pip install [--editable] . ``` ### Tests diff --git a/examples/README.md b/examples/README.md index 622fa07f8f..0c57990ea7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,12 +4,12 @@ In this section a few examples are put together. All of these examples work for similar API between the different models. **Important** -To use the examples, execute the following steps in a new virtual environment: +To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: ```bash git clone git@github.com:huggingface/transformers cd transformers -pip install . +pip install [--editable] . ``` | Section | Description | From afaa33585109550f9ecaaee4e47f187aaaefedd0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 23 Nov 2019 11:34:45 -0500 Subject: [PATCH 14/91] [doc] Fix assets urls --- docs/source/_static/js/custom.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index 2c7836fd20..ec804b3704 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -1,5 +1,5 @@ function addIcon() { - const huggingFaceLogo = "https://huggingface.co/assets/transformers-docs/huggingface_logo.svg"; + const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg"; const image = document.createElement("img"); image.setAttribute("src", huggingFaceLogo); @@ -24,10 +24,10 @@ function addCustomFooter() { social.classList.add("footer__Social"); const imageDetails = [ - { link: "https://huggingface.co", imageLink: "https://huggingface.co/assets/transformers-docs/website.svg" }, - { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/twitter.svg" }, - { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/github.svg" }, - { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/assets/transformers-docs/linkedin.svg" } + { link: "https://huggingface.co", imageLink: "https://huggingface.co/landing/assets/transformers-docs/website.svg" }, + { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/twitter.svg" }, + { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/github.svg" }, + { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/landing/assets/transformers-docs/linkedin.svg" } ]; imageDetails.forEach(imageLinks => { From 7485caefb09cb7f4c4b720b40ec69fed345a6b1c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 25 Nov 2019 09:33:39 -0500 Subject: [PATCH 15/91] fix #1894 --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 52a1b75a65..aded521c1d 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -68,7 +68,7 @@ class TextDataset(Dataset): directory, filename = os.path.split(file_path) cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename) - if os.path.exists(cached_features_file): + if os.path.exists(cached_features_file) and not args.overwrite_cache: logger.info("Loading features from cached file %s", cached_features_file) with open(cached_features_file, 'rb') as handle: self.examples = pickle.load(handle) From 99f750d64e78d20fa5213ea11235b6a1b084481e Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 10:35:07 +0100 Subject: [PATCH 16/91] add Camembert models to modeling_auto --- transformers/modeling_auto.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..a5828148ad 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -27,6 +27,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice from .modeling_utils import PreTrainedModel, SequenceSummary @@ -48,6 +49,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -71,6 +73,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -138,6 +141,8 @@ class AutoModel(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -172,6 +177,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -198,6 +204,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -264,6 +271,8 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + if 'camembert' in pretrained_model_name_or_path: + return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -298,6 +307,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -320,6 +330,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -383,6 +394,8 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + if 'camembert' in pretrained_model_name_or_path: + return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: From c8eb8157b86291413fe8096217f4defc168aa73f Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 11:01:20 +0100 Subject: [PATCH 17/91] fix docstrings --- transformers/modeling_auto.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index a5828148ad..ce36f6dc4a 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -49,7 +49,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -73,7 +73,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -177,7 +177,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -204,7 +204,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -307,7 +307,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -330,7 +330,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) From fa963ecc59a1dea59c2d0e952b2c4483e1828176 Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 11:03:12 +0100 Subject: [PATCH 18/91] =?UTF-8?q?if=E2=86=92elif?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- transformers/modeling_auto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index ce36f6dc4a..5866420001 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -271,7 +271,7 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - if 'camembert' in pretrained_model_name_or_path: + elif 'camembert' in pretrained_model_name_or_path: return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) @@ -394,7 +394,7 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - if 'camembert' in pretrained_model_name_or_path: + elif 'camembert' in pretrained_model_name_or_path: return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) From 07bf43074f39379d8c7f6afcca105e69685f7531 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Thu, 21 Nov 2019 20:52:06 -0600 Subject: [PATCH 19/91] Fix GPT2 docstring --- transformers/tokenization_gpt2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index a4798b0129..5fda709448 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -107,10 +107,10 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ GPT-2 BPE tokenizer. Peculiarities: - Byte-level Byte-Pair-Encoding - - Requires a space to start the input string => the encoding methods should be called with the + - Requires a space to start the input string => the encoding and tokenize methods should be called with the ``add_prefix_space`` flag set to ``True``. - Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve - the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"` + Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve + the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(" Hello")) = "Hello"` """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP From aa92a184d2b92faadec975139ad55e2ae749362c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0brahim=20Ethem=20Demirci?= Date: Sat, 16 Nov 2019 16:49:37 +0300 Subject: [PATCH 20/91] resize model when special tokenizer present --- examples/run_lm_finetuning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index aded521c1d..c33aa94a32 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -215,6 +215,7 @@ def train(args, train_dataset, model, tokenizer): global_step = 0 tr_loss, logging_loss = 0.0, 0.0 + model.resize_token_embeddings(len(tokenizer)) model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) From 5d3b8daad2cc6287d30f03f8a96d0a1f7bc8d0dc Mon Sep 17 00:00:00 2001 From: manansanghi <52307004+manansanghi@users.noreply.github.com> Date: Fri, 22 Nov 2019 10:31:54 -0800 Subject: [PATCH 21/91] Minor bug fixes on run_ner.py --- examples/run_ner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 127d63a6cd..1ab1236d94 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -127,7 +127,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id): "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -217,7 +217,7 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix="" "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] From 8e5d84fcc1a645d3c13b8a2f64fa995637440dad Mon Sep 17 00:00:00 2001 From: v_sboliu Date: Tue, 26 Nov 2019 12:04:31 +0800 Subject: [PATCH 22/91] Fixed typo --- transformers/modeling_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 7c2c6f4602..81d92d8f1b 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -278,7 +278,7 @@ class BertAttention(nn.Module): if len(heads) == 0: return mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + heads = set(heads) - self.pruned_heads # Convert to set and remove already pruned heads for head in heads: # Compute how many pruned heads are before the head and move the index accordingly head = head - sum(1 if h < head else 0 for h in self.pruned_heads) From c0c2088333e2e8ce2b24d0c7f4bf071dcccbd7ea Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 19:57:38 +0000 Subject: [PATCH 23/91] ALBERT model --- transformers/configuration_albert.py | 72 ++++ ...lbert_original_tf_checkpoint_to_pytorch.py | 44 +++ transformers/modeling_albert.py | 331 ++++++++++++++++++ 3 files changed, 447 insertions(+) create mode 100644 transformers/configuration_albert.py create mode 100644 transformers/convert_albert_original_tf_checkpoint_to_pytorch.py create mode 100644 transformers/modeling_albert.py diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py new file mode 100644 index 0000000000..c86c9565cb --- /dev/null +++ b/transformers/configuration_albert.py @@ -0,0 +1,72 @@ +from .configuration_utils import PretrainedConfig + +class AlbertConfig(PretrainedConfig): + """Configuration for `AlbertModel`. + + The default settings match the configuration of model `albert_xxlarge`. + """ + + def __init__(self, + vocab_size_or_config_json_file, + embedding_size=128, + hidden_size=4096, + num_hidden_layers=12, + num_hidden_groups=1, + num_attention_heads=64, + intermediate_size=16384, + inner_group_num=1, + down_scale_factor=1, + hidden_act="gelu", + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, **kwargs): + """Constructs AlbertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `AlbertModel`. + embedding_size: size of voc embeddings. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_hidden_groups: Number of group for the hidden layers, parameters in + the same group are shared. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + inner_group_num: int, number of inner repetition of attention and ffn. + down_scale_factor: float, the scale to apply + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `AlbertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + super(AlbertConfig, self).__init__(**kwargs) + + self.vocab_size = vocab_size_or_config_json_file + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_hidden_groups = num_hidden_groups + self.num_attention_heads = num_attention_heads + self.inner_group_num = inner_group_num + self.down_scale_factor = down_scale_factor + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps \ No newline at end of file diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py new file mode 100644 index 0000000000..04877d41b9 --- /dev/null +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,44 @@ + + + +from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert + + + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + # Initialise PyTorch model + config = BertConfig.from_json_file(bert_config_file) + print("Building PyTorch model from configuration: {}".format(str(config))) + model = BertForPreTraining(config) + + # Load weights from tf checkpoint + load_tf_weights_in_bert(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--albert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained BERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, + args.bert_config_file, + args.pytorch_dump_path) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py new file mode 100644 index 0000000000..b006cbe8fd --- /dev/null +++ b/transformers/modeling_albert.py @@ -0,0 +1,331 @@ + +import os +import math +import logging +import torch +import torch.nn as nn +from transformers.configuration_albert import AlbertConfig +logger = logging.getLogger(__name__) + +def load_tf_weights_in_albert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model.""" + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + print(model) + + for name, array in zip(names, arrays): + og = name + name = name.replace("transformer/group_0/inner_group_0", "transformer") + name = name.replace("LayerNorm", "layer_norm") + name = name.replace("ffn_1", "ffn") + name = name.replace("ffn/intermediate/output", "ffn_output") + name = name.replace("attention_1", "attention") + name = name.replace("cls/predictions/transform", "predictions") + name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm") + name = name.split('/') + + print(name) + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + print("transposed") + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + print("Initialize PyTorch weight {} from {}".format(name, og)) + pointer.data = torch.from_numpy(array) + + return model + + +class AlbertEmbeddings(nn.Module): + """ + Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(AlbertEmbeddings, self).__init__() + + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) + self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + word_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = word_embeddings + position_embeddings + token_type_embeddings + embeddings = self.layer_norm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + + def get_word_embeddings_table(self): + return self.word_embeddings + + +class AlbertModel(nn.Module): + def __init__(self, config): + super(AlbertModel, self).__init__() + + self.config = config + self.embeddings = AlbertEmbeddings(config) + self.encoder = AlbertEncoder(config) + self.pooler = nn.Linear(config.hidden_size, config.hidden_size) + self.pooler_activation = nn.Tanh() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + sequence_output = encoder_outputs[0] + print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs + + +class AlbertForMaskedLM(nn.Module): + def __init__(self, config): + super(AlbertForMaskedLM, self).__init__() + + self.config = config + self.bert = AlbertModel(config) + self.layer_norm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.classifier.word_embeddings, + self.transformer.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] + hidden_states = self.dense(hidden_states) + hidden_states = gelu_new(hidden_states) + hidden_states = self.layer_norm(hidden_states) + + logits = self.word_embeddings(hidden_states) + + return logits + + +class AlbertAttention(nn.Module): + def __init__(self, config): + super(AlbertAttention, self).__init__() + + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + w = self.dense.weight.T.view(16, 64, 1024) + b = self.dense.bias + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) + return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config =config + self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for i in range(self.config.num_hidden_layers): + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.layer_norm(ffn_output + attention_output) + + return hidden_states + + +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +class AlbertEncoder(nn.Module): + def __init__(self, config): + super(AlbertEncoder, self).__init__() + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.transformer = AlbertTransformer(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + hidden_states = self.transformer(hidden_states, attention_mask, head_mask) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + +# config = AlbertConfig.from_json_file("config.json") +# # model = AlbertForMaskedLM(config) +# model = AlbertModel(config) + +# model = load_tf_weights_in_albert(model, config, "albert/albert") + +# print(model) + +# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]]) +# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]]) +# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]]) + +# # sequence_output, pooled_outputs = model() + +# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1] + + +# embeddings_output = +# print("pooled output", logits) +# # print("Pooled output", pooled_outputs) + +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json") +model = AlbertModel(config) +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert") \ No newline at end of file From 91ccbae788de11f0f5ffb862421e345b27a20a76 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 21:21:57 +0000 Subject: [PATCH 24/91] Accepts multiple sizes --- transformers/modeling_albert.py | 143 ++++++++++++++------------------ 1 file changed, 60 insertions(+), 83 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b006cbe8fd..f3cebdc3d9 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -5,6 +5,7 @@ import logging import torch import torch.nn as nn from transformers.configuration_albert import AlbertConfig +from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new logger = logging.getLogger(__name__) def load_tf_weights_in_albert(model, config, tf_checkpoint_path): @@ -32,14 +33,14 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): print(model) for name, array in zip(names, arrays): + print(name) og = name name = name.replace("transformer/group_0/inner_group_0", "transformer") - name = name.replace("LayerNorm", "layer_norm") name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm") + name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm") name = name.split('/') print(name) @@ -84,44 +85,22 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): return model -class AlbertEmbeddings(nn.Module): +class AlbertEmbeddings(BertEmbeddings): """ Construct the embeddings from word, position and token_type embeddings. """ def __init__(self, config): - super(AlbertEmbeddings, self).__init__() + super(AlbertEmbeddings, self).__init__(config) self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) - self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - def forward(self, input_ids, token_type_ids=None, position_ids=None): - seq_length = input_ids.size(1) - if position_ids is None: - position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) - if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) - - word_embeddings = self.word_embeddings(input_ids) - position_embeddings = self.position_embeddings(position_ids) - token_type_embeddings = self.token_type_embeddings(token_type_ids) - - embeddings = word_embeddings + position_embeddings + token_type_embeddings - embeddings = self.layer_norm(embeddings) - embeddings = self.dropout(embeddings) - return embeddings + self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) - def get_word_embeddings_table(self): - return self.word_embeddings - - -class AlbertModel(nn.Module): +class AlbertModel(BertModel): def __init__(self, config): - super(AlbertModel, self).__init__() + super(AlbertModel, self).__init__(config) self.config = config self.embeddings = AlbertEmbeddings(config) @@ -129,6 +108,7 @@ class AlbertModel(nn.Module): self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) @@ -166,7 +146,7 @@ class AlbertForMaskedLM(nn.Module): self.config = config self.bert = AlbertModel(config) - self.layer_norm = nn.LayerNorm(config.embedding_size) + self.LayerNorm = nn.LayerNorm(config.embedding_size) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) @@ -182,39 +162,47 @@ class AlbertForMaskedLM(nn.Module): hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] hidden_states = self.dense(hidden_states) hidden_states = gelu_new(hidden_states) - hidden_states = self.layer_norm(hidden_states) + hidden_states = self.LayerNorm(hidden_states) logits = self.word_embeddings(hidden_states) return logits -class AlbertAttention(nn.Module): +class AlbertAttention(BertSelfAttention): def __init__(self, config): - super(AlbertAttention, self).__init__() - - if config.hidden_size % config.num_attention_heads != 0: - raise ValueError( - "The hidden size (%d) is not a multiple of the number of attention " - "heads (%d)" % (config.hidden_size, config.num_attention_heads)) - self.output_attentions = config.output_attentions + super(AlbertAttention, self).__init__(config) self.num_attention_heads = config.num_attention_heads - self.attention_head_size = int(config.hidden_size / config.num_attention_heads) - self.all_head_size = self.num_attention_heads * self.attention_head_size - - self.query = nn.Linear(config.hidden_size, self.all_head_size) - self.key = nn.Linear(config.hidden_size, self.all_head_size) - self.value = nn.Linear(config.hidden_size, self.all_head_size) - + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads self.dropout = nn.Dropout(config.attention_probs_dropout_prob) self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() - def transpose_for_scores(self, x): - new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) - x = x.view(*new_x_shape) - return x.permute(0, 2, 1, 3) + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) @@ -248,7 +236,8 @@ class AlbertAttention(nn.Module): context_layer = context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) reshaped_context_layer = context_layer.view(*new_context_layer_shape) - w = self.dense.weight.T.view(16, 64, 1024) + print(self.dense.weight.T.shape) + w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) b = self.dense.bias projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b @@ -262,7 +251,7 @@ class AlbertTransformer(nn.Module): super(AlbertTransformer, self).__init__() self.config =config - self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) @@ -273,18 +262,11 @@ class AlbertTransformer(nn.Module): ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.layer_norm(ffn_output + attention_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) return hidden_states -def gelu_new(x): - """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). - Also see https://arxiv.org/abs/1606.08415 - """ - return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) - - class AlbertEncoder(nn.Module): def __init__(self, config): super(AlbertEncoder, self).__init__() @@ -305,27 +287,22 @@ class AlbertEncoder(nn.Module): outputs = outputs + (all_attentions,) return outputs # last-layer hidden state, (all hidden states), (all attentions) -# config = AlbertConfig.from_json_file("config.json") -# # model = AlbertForMaskedLM(config) -# model = AlbertModel(config) - -# model = load_tf_weights_in_albert(model, config, "albert/albert") - -# print(model) - -# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]]) -# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]]) -# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]]) - -# # sequence_output, pooled_outputs = model() - -# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1] - - -# embeddings_output = -# print("pooled output", logits) -# # print("Pooled output", pooled_outputs) - -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json") +model_size = "base" +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size)) model = AlbertModel(config) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert") \ No newline at end of file +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size)) +model.eval() +print(sum(p.numel() for p in model.parameters() if p.requires_grad)) + + +input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] +input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] +segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] + +pt_input_ids = torch.tensor(input_ids) +pt_input_mask = torch.tensor(input_mask) +pt_segment_ids = torch.tensor(segment_ids) + +pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} +pt_output = model(**pt_dict) +print(pt_output) \ No newline at end of file From 139affaa8de89c6ef16a4712d24a22a8fea91eda Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 22:45:24 +0000 Subject: [PATCH 25/91] Albert layer/layer groups --- transformers/modeling_albert.py | 80 ++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index f3cebdc3d9..440ccf2bce 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -30,17 +30,19 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): names.append(name) arrays.append(array) - print(model) + for name, array in zip(names, arrays): + print(name) for name, array in zip(names, arrays): print(name) og = name - name = name.replace("transformer/group_0/inner_group_0", "transformer") name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm") + name = name.replace("LayerNorm_1", "attention/LayerNorm") + name = name.replace("inner_group_", "albert_layers/") + name = name.replace("group_", "albert_layer_groups/") name = name.split('/') print(name) @@ -104,7 +106,7 @@ class AlbertModel(BertModel): self.config = config self.embeddings = AlbertEmbeddings(config) - self.encoder = AlbertEncoder(config) + self.encoder = AlbertTransformer(config) self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() @@ -133,6 +135,7 @@ class AlbertModel(BertModel): extended_attention_mask, head_mask=head_mask) sequence_output = encoder_outputs[0] + print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) @@ -246,18 +249,18 @@ class AlbertAttention(BertSelfAttention): return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask -class AlbertTransformer(nn.Module): +class AlbertLayer(nn.Module): def __init__(self, config): - super(AlbertTransformer, self).__init__() + super(AlbertLayer, self).__init__() - self.config =config + self.config = config self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for i in range(self.config.num_hidden_layers): + for _ in range(self.config.inner_group_num): attention_output = self.attention(hidden_states, attention_mask)[0] ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) @@ -267,42 +270,59 @@ class AlbertTransformer(nn.Module): return hidden_states -class AlbertEncoder(nn.Module): +class AlbertLayerGroup(nn.Module): def __init__(self, config): - super(AlbertEncoder, self).__init__() + super(AlbertLayerGroup, self).__init__() + + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for albert_layer in self.albert_layers: + hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + + return hidden_states + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) - self.transformer = AlbertTransformer(config) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) def forward(self, hidden_states, attention_mask=None, head_mask=None): hidden_states = self.embedding_hidden_mapping_in(hidden_states) - hidden_states = self.transformer(hidden_states, attention_mask, head_mask) - outputs = (hidden_states,) - if self.output_hidden_states: - outputs = outputs + (all_hidden_states,) - if self.output_attentions: - outputs = outputs + (all_attentions,) - return outputs # last-layer hidden state, (all hidden states), (all attentions) + for layer_idx in range(self.config.num_hidden_layers): + group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + + return (hidden_states,) -model_size = "base" -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size)) + +model_size = 'base' +hidden_groups = 1 +inner_groups = 1 +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) model = AlbertModel(config) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size)) + +print(model) +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) model.eval() print(sum(p.numel() for p in model.parameters() if p.requires_grad)) -input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] -input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] -segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] +# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] +# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] +# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] -pt_input_ids = torch.tensor(input_ids) -pt_input_mask = torch.tensor(input_mask) -pt_segment_ids = torch.tensor(segment_ids) +# pt_input_ids = torch.tensor(input_ids) +# pt_input_mask = torch.tensor(input_mask) +# pt_segment_ids = torch.tensor(segment_ids) -pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} -pt_output = model(**pt_dict) -print(pt_output) \ No newline at end of file +# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} +# pt_output = model(**pt_dict) +# print(pt_output) \ No newline at end of file From 12290c0d5ce8475e884190b6ba480a8b3e671b3e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:19:02 +0000 Subject: [PATCH 26/91] Handles multi layer and multi groups --- transformers/modeling_albert.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 440ccf2bce..90e9b162e6 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -136,7 +136,6 @@ class AlbertModel(BertModel): head_mask=head_mask) sequence_output = encoder_outputs[0] - print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here @@ -260,12 +259,11 @@ class AlbertLayer(nn.Module): self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for _ in range(self.config.inner_group_num): - attention_output = self.attention(hidden_states, attention_mask)[0] - ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) - ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) return hidden_states @@ -303,16 +301,16 @@ class AlbertTransformer(nn.Module): return (hidden_states,) -model_size = 'base' -hidden_groups = 1 -inner_groups = 1 -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) -model = AlbertModel(config) +# model_size = 'base' +# hidden_groups = 1 +# inner_groups = 2 +# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) +# model = AlbertModel(config) -print(model) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) -model.eval() -print(sum(p.numel() for p in model.parameters() if p.requires_grad)) +# # print(model) +# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) +# # model.eval() +# # print(sum(p.numel() for p in model.parameters() if p.requires_grad)) # input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] From 1b92564330aa2f40b065a0b9a2a94a28a595bbc6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:23:18 +0000 Subject: [PATCH 27/91] Reorganize and cleanup --- transformers/modeling_albert.py | 287 +++++++++++++++----------------- 1 file changed, 132 insertions(+), 155 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 90e9b162e6..c6662cb6d3 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -100,6 +100,138 @@ class AlbertEmbeddings(BertEmbeddings): self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) +class AlbertAttention(BertSelfAttention): + def __init__(self, config): + super(AlbertAttention, self).__init__(config) + + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + + + # Should find a better way to do this + w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) + b = self.dense.bias + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) + return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + + +class AlbertLayer(nn.Module): + def __init__(self, config): + super(AlbertLayer, self).__init__() + + self.config = config + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) + + return hidden_states + + +class AlbertLayerGroup(nn.Module): + def __init__(self, config): + super(AlbertLayerGroup, self).__init__() + + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for albert_layer in self.albert_layers: + hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + + return hidden_states + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + + for layer_idx in range(self.config.num_hidden_layers): + group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + + return (hidden_states,) + + class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -169,158 +301,3 @@ class AlbertForMaskedLM(nn.Module): logits = self.word_embeddings(hidden_states) return logits - - -class AlbertAttention(BertSelfAttention): - def __init__(self, config): - super(AlbertAttention, self).__init__(config) - - self.num_attention_heads = config.num_attention_heads - self.hidden_size = config.hidden_size - self.attention_head_size = config.hidden_size // config.num_attention_heads - self.dropout = nn.Dropout(config.attention_probs_dropout_prob) - self.dense = nn.Linear(config.hidden_size, config.hidden_size) - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.pruned_heads = set() - - def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.num_attention_heads, self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - - # Prune linear layers - self.query = prune_linear_layer(self.query, index) - self.key = prune_linear_layer(self.key, index) - self.value = prune_linear_layer(self.value, index) - self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) - - # Update hyper params and store pruned heads - self.num_attention_heads = self.num_attention_heads - len(heads) - self.all_head_size = self.attention_head_size * self.num_attention_heads - self.pruned_heads = self.pruned_heads.union(heads) - - def forward(self, input_ids, attention_mask=None, head_mask=None): - mixed_query_layer = self.query(input_ids) - mixed_key_layer = self.key(input_ids) - mixed_value_layer = self.value(input_ids) - - query_layer = self.transpose_for_scores(mixed_query_layer) - key_layer = self.transpose_for_scores(mixed_key_layer) - value_layer = self.transpose_for_scores(mixed_value_layer) - - # Take the dot product between "query" and "key" to get the raw attention scores. - attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) - attention_scores = attention_scores / math.sqrt(self.attention_head_size) - if attention_mask is not None: - # Apply the attention mask is (precomputed for all layers in BertModel forward() function) - attention_scores = attention_scores + attention_mask - - # Normalize the attention scores to probabilities. - attention_probs = nn.Softmax(dim=-1)(attention_scores) - - # This is actually dropping out entire tokens to attend to, which might - # seem a bit unusual, but is taken from the original Transformer paper. - attention_probs = self.dropout(attention_probs) - - # Mask heads if we want to - if head_mask is not None: - attention_probs = attention_probs * head_mask - - context_layer = torch.matmul(attention_probs, value_layer) - - context_layer = context_layer.permute(0, 2, 1, 3).contiguous() - new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) - reshaped_context_layer = context_layer.view(*new_context_layer_shape) - print(self.dense.weight.T.shape) - w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) - b = self.dense.bias - - projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b - projected_context_layer = self.dropout(projected_context_layer) - layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) - return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask - - -class AlbertLayer(nn.Module): - def __init__(self, config): - super(AlbertLayer, self).__init__() - - self.config = config - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.attention = AlbertAttention(config) - self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) - self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask)[0] - ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) - ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) - - return hidden_states - - -class AlbertLayerGroup(nn.Module): - def __init__(self, config): - super(AlbertLayerGroup, self).__init__() - - self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - for albert_layer in self.albert_layers: - hidden_states = albert_layer(hidden_states, attention_mask, head_mask) - - return hidden_states - - -class AlbertTransformer(nn.Module): - def __init__(self, config): - super(AlbertTransformer, self).__init__() - - self.config = config - self.output_attentions = config.output_attentions - self.output_hidden_states = config.output_hidden_states - self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) - self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - hidden_states = self.embedding_hidden_mapping_in(hidden_states) - - for layer_idx in range(self.config.num_hidden_layers): - group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) - - return (hidden_states,) - - -# model_size = 'base' -# hidden_groups = 1 -# inner_groups = 2 -# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) -# model = AlbertModel(config) - -# # print(model) -# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) -# # model.eval() -# # print(sum(p.numel() for p in model.parameters() if p.requires_grad)) - - -# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] -# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] -# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] - -# pt_input_ids = torch.tensor(input_ids) -# pt_input_mask = torch.tensor(input_mask) -# pt_segment_ids = torch.tensor(segment_ids) - -# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} -# pt_output = model(**pt_dict) -# print(pt_output) \ No newline at end of file From 67b422662c7002319e54679a73d654ba313702ef Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:33:53 +0000 Subject: [PATCH 28/91] Documentation + improved AlbertForMaskedLM --- transformers/modeling_albert.py | 85 +++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index c6662cb6d3..9b3c51fe25 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -4,8 +4,11 @@ import math import logging import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from .file_utils import add_start_docstrings + logger = logging.getLogger(__name__) def load_tf_weights_in_albert(model, config, tf_checkpoint_path): @@ -232,6 +235,70 @@ class AlbertTransformer(nn.Module): return (hidden_states,) +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -274,6 +341,7 @@ class AlbertModel(BertModel): return outputs +@add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(nn.Module): def __init__(self, config): super(AlbertForMaskedLM, self).__init__() @@ -292,12 +360,19 @@ class AlbertForMaskedLM(nn.Module): self._tie_or_clone_weights(self.classifier.word_embeddings, self.transformer.embeddings.word_embeddings) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): - hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] - hidden_states = self.dense(hidden_states) + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None): + outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + sequence_outputs = outputs[0] + hidden_states = self.dense(sequence_outputs) hidden_states = gelu_new(hidden_states) hidden_states = self.LayerNorm(hidden_states) + prediction_scores = self.word_embeddings(hidden_states) - logits = self.word_embeddings(hidden_states) + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs - return logits + return outputs From fedac786d401a9da31847b45df842bbee185afe6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 14:39:16 +0000 Subject: [PATCH 29/91] Tokenization + small fixes --- transformers/modeling_albert.py | 2 +- transformers/tokenization_albert.py | 210 ++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 transformers/tokenization_albert.py diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 9b3c51fe25..a1c0d5f610 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -298,7 +298,7 @@ ALBERT_INPUTS_DOCSTRING = r""" """ @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", - BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py new file mode 100644 index 0000000000..f2e37222f6 --- /dev/null +++ b/transformers/tokenization_albert.py @@ -0,0 +1,210 @@ + +from .tokenization_utils import PreTrainedTokenizer +import logging +import unicodedata +import six +import os +from shutil import copyfile + +logger = logging.getLogger(__name__) + +SPIECE_UNDERLINE = u'▁' + +class AlbertTokenizer(PreTrainedTokenizer): + """ + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + # vocab_files_names = VOCAB_FILES_NAMES + # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, + do_lower_case=False, remove_space=True, keep_accents=False, + bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", + pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): + super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, + unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + + self.do_lower_case = do_lower_case + self.remove_space = remove_space + self.keep_accents = keep_accents + self.vocab_file = vocab_file + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + + @property + def vocab_size(self): + return len(self.sp_model) + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(self.vocab_file) + + def preprocess_text(self, inputs): + if self.remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + outputs = outputs.replace("``", '"').replace("''", '"') + + if six.PY2 and isinstance(outputs, str): + outputs = outputs.decode('utf-8') + + if not self.keep_accents: + outputs = unicodedata.normalize('NFKD', outputs) + outputs = ''.join([c for c in outputs if not unicodedata.combining(c)]) + if self.do_lower_case: + outputs = outputs.lower() + + return outputs + + def _tokenize(self, text, return_unicode=True, sample=False): + """ Tokenize a string. + return_unicode is used only for py2 + """ + text = self.preprocess_text(text) + # note(zhiliny): in some systems, sentencepiece only accepts str for py2 + if six.PY2 and isinstance(text, unicode): + text = text.encode('utf-8') + + if not sample: + pieces = self.sp_model.EncodeAsPieces(text) + else: + pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + cur_pieces = self.sp_model.EncodeAsPieces( + piece[:-1].replace(SPIECE_UNDERLINE, '')) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + # note(zhiliny): convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in new_pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + new_pieces = ret_pieces + + return new_pieces + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + return self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index, return_unicode=True): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + token = self.sp_model.IdToPiece(index) + if six.PY2 and return_unicode and isinstance(token, str): + token = token.decode('utf-8') + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return token_ids_0 + sep + cls + return token_ids_0 + sep + token_ids_1 + sep + cls + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1] + return ([0] * len(token_ids_0)) + [1, 1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 + | first sequence | second sequence | CLS segment ID + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + cls_segment_id = [2] + + if token_ids_1 is None: + return len(token_ids_0 + sep + cls) * [0] + return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From e3ea5d1d8db36091ab840328cf6691c4f46f2e89 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 15:03:30 +0000 Subject: [PATCH 30/91] Docstrings --- transformers/modeling_albert.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index a1c0d5f610..ad8b979cef 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -300,6 +300,25 @@ ALBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -343,6 +362,27 @@ class AlbertModel(BertModel): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(nn.Module): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ + def __init__(self, config): super(AlbertForMaskedLM, self).__init__() From ee20201d339b62948acd05783efc48c4e2b466bb Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 16:19:49 +0000 Subject: [PATCH 31/91] Tokenization tests + fixes + init --- transformers/__init__.py | 5 ++ transformers/tests/fixtures/30k-clean.model | Bin 0 -> 760289 bytes .../tests/tokenization_albert_test.py | 78 ++++++++++++++++++ transformers/tokenization_albert.py | 30 +++---- transformers/tokenization_xlnet.py | 8 +- 5 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 transformers/tests/fixtures/30k-clean.model create mode 100644 transformers/tests/tokenization_albert_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 5c7b0a6197..152d520e7b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_albert import AlbertTokenizer from .tokenization_camembert import CamembertTokenizer # Configurations @@ -57,6 +58,8 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import AlbertConfig, ALBERT +from .configuration_albert import AlbertConfig from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling @@ -104,6 +107,8 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model + from .modeling_albert import (AlbertModel, AlbertForMaskedLM) + # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/30k-clean.model new file mode 100644 index 0000000000000000000000000000000000000000..c91b8acfa56ccfc80e1cdd854ddcaf9b6c44ab2a GIT binary patch literal 760289 zcmZUcd7RhN_s5k&L$Y)#Q6X7UnrY9TB`SONgwHlJpPA3r=l%K2G+9#FLzbju2}KB5 z$`(rYuO&r<5?PW$2?_Z<&pq$Y9goNF^VdDE&t1+v_uTE=bML!s=d$L9=OXbVWgC9& zwk=b-n5hlvrb-V#_M~AW{`3FvVaK25=~iXi4L^C{u%R9(D{FF~MQ&C`_@MuU_x(?} z`+vfRl)^ieZTs~Vx0R?J%bLFWK#9=5#?3BKWl)pHO8a&@EhuYQwjJR@bwc9fj$2T+ zLs^qmS-d* zyS00z^k;P>E-fc`vLRcP^H)HUNnC#ZY(XhiG$B_vUFh&b<-!54mWr)U4bbd2dY)s4l4WVWUN|(=K-uSsv~s?xpJ@f90w6;dKn@62YqXhE3UJD zj*@F85VA^CC)3hx;rlL0LcZK)iT{I!KBOa1Ko z12jZ^8Ut8O2uXp^&)>m z()ek)qwS}qoTvw_6`ctXcGdcm_#^qH)5&aNz_PiJ9v zY5A8W?qX>CuahCER)h5VnUGh2jJ$lws!}%f33=!iDB6P#{P?zCl_EwXv0B-2zNhQ8 zLq8Y!+C`Qh~xv=F2AnS zg6RmN8wW|RK{~R>7lbYIc(y(x`~3}F(!D&ow_W@7r3mRPbxEBJZKX#Og?jBM*@Zqz zeUffF<5OtU?o75`X4U>w%CeY=qi$R8^#Bz}FdWYH0E12{A$?Z>7+3WPa%{cR&rTRU zrB#}c{S!+T8l?Sazm(>J3fcSMUolBTiN_q$8k)~JeH6ZpN?c2zU-O6^P*)x$sGMkB{WZ0!( zmSvELWvSJOk9;a7puBY3ZPS9X)@3b8Rm>zAJOY(kurH|P9c2iRp;ex)PB4viUJh-7 zL`ecvYPa7?2~?$I+X2uz*alKHjG&379_8w_5Ykk!Dl3<4_j@U$Vy2Kx$xm$nEOu3b zsr(Q~x>bcd+>4O0l&C|g9vSXas`N85c)CxSt*n$??)N0(k^e3B$?LQETG@Mvrz?}G zv@C1&M@dh5%58^3SXqTcBrSWyp{=?BOr~U?w|#_E7FD?POXyM^^YUen&8`D!!8>0_ zxYV~}5yp-)MuUX8B{Q*vG#%FnF#cA{QCImiG)e*8sF(<1QY!}X#-C8eUv-l1z2a|Z zE4~^QEU;CmBPJ_{t}%&T0Vzn{ROqj?#@9LeR3z!(Bn{y9gj`x&}0&_b7|YYGKF?1dXqN zj{f7jknmM2_Z{$8smLG^39=bE;s}tiQL@=u**qB9ipga2NNv&OK0+#4nUIN#8ne(2 z7s$PDfY`#Ymz=%HN6%=pyx8jRQvQL`nSP#5MCiX4PJ*`1h>+32ylkEdWZcJ5?XCAh z8b`TQgogd&X#jcwA{_mi1yP<&iwRmq(mF8A*y8DGomAPcxcjWNRBwSL?pu;QpRYB`OPY%aPFGSjC!BmkrI&12VOVv0OuqcTa0*$KNv|J#4(SL>fa6>ITPieD zOh;j3kbszA-0a)B%G0R`L+Y*LppElbHl5S{G2}`Q1=`YR^@y8%s+w$dMjrkE8bS7w zCzca3o)ZXWq{Ci|+U%h*e$(;e(a5I@Cup1zETS z+9VK7RM8KHZQG=jYd(>pXOl;#GB7H}5_umX2%}YUY6lDTk$hJ|R%j|&C@?RM831C1 z7P48{coZb*85z$yfuQy9-qQU(ke(0(9QuX*Y2kxqZ`8aS=8<2k#AOQ+Jgv`!neT67PF3sBs;nb(2&z$OGv} zp7G|MD1fn9&^BvU1#LVO<-WlG$w(8GifPc+IS>MC$%2e|3<#aoQK0f&!n9XW=By$V z){WtrWbPta}4c5%ZPt1rJwxvc57rHC~}BGxzebiG#Ti5;MghpI%z z^~Mj&e3U?^ym#9{TZae|xkAEq{KxkJgt^|bv3uj+46)i8Rnnq2pv|eQxX(Eh%H$St zbNU`f16boCSQ#=1(rVM73y6_}p~*Zar=3p7G;;}Dijy<1`iIRMKhU_tX)BU#BF za7mtjt!mj=A*;hx?}CSPoybB?4jK<+W!DtZyPaL;uNF=nHxBKG#dh}0UlDY_7? zRwMf#0A^)Ulh?XInW!qMs*JDjDdLHkRpgxBK1D?zIqe9aBASgi$dsd?tx$G+Y?jJT z{7(c{e6ESpU%Cj|W#cSP075OH4h#sNddV! zO;)*lLYE>;a+BZpk4lYRi!+zk&v}WB+$8c z`)#%_DL|o^&wz~B76kqnF}jx#wAeXzGI_aO0V_)_9WB~GYrI~rXQG9Jf5tq#1uxwA221uRWv0+m^t@M&TcTVxJc)2_eHc0{K2k3) zJ>tn6gV#lKAx%^^KL^t76(BY5!R-9S7RK%)&3+^t%DsTutsuYr>=SdiP{<;)Q~w3B z(yJ4xoa`VwloF+F*Y{&PkWd%vBedtw9RNa2hQSNUeF_c?w7L%|Vx6#`yj0sx8Z5p=r9uRAYc(N`T=sfr2 z3@~Gr4H%r!$nIYP zguPrBT$~MI$N2%!n!~p830bxt$ZFqT+HQa_1`4{4t5w=^O}RBeQ32}6>B#C9ElU6C zjy#a(S^|UvX@42o8p?{oM4c(ncmvx2S?(!anr*wMr|C3tIiowYm6qj1BO}M01Yi|r zO&mvL+0#KxA~8%8(;7+L-+z8K|5}+rE)mlWd;5q_P+#33HF=+7CEcIZ;FvleAf(UyncG2yG%`%sBLB|8GIf$`2`R@<4=)jMb9>tW6`a7}J-k?`|`EtVmi{g1bKIv*}sR z%JUwjQIdK|KW3p3Qu!<)Q^zb;GB;#p7J*rDY-%WO?%M#yePt1T4td{4sA53P$Xg#a zMqrgH)ytqQ*HFDsA;rQGfLTrTzV$agS`;HnCV?h@{e#a6TUH?JmA0m55t3rpandG_ zBr@(;VdZZis*HJ=+oG%_$co-_{_apFC)U1cHDWf%+_r$B_UWqd^Sz)O6*R_rf7*UP zR;aTt)rA<^cVRh5m?52Btv1K>29mO!*>lF}(Wz|C zZIb^!1~}9bj1pHI2Wf(@&#KY(`V#=CSf0b|GYA^j9ARhVw_(sGV&-mUBUE^=v%wfq zb@;D6*T+xkdB|fIKwI8v*MrAj1VG+wLPkd{m`F*7%LrP{vDxZ=q3UuFQ_XZzy4(mA zszQ?OYX3==v=EWWVUQE6in~FrUKx0aJ?68h=q2+PSdw%^9$rk)a>MMJk!wHkG#XnW z-zeYE>ey@=g3= zQIip=+vxv{q%Z+8Vl>;a#2iBeHsa2nuGabFg#)0Cmt^3^u{KBiCkFlGos*%%VW^nP zX;)f)IuOiZURZgyPajFSGpno41F`C|*<(oQ838Cw7A#+w`ec`ck7Nf#6Ker%SQ;Qd z6(B;17HOZcS9ltU_LSyRp^a&0eF&sR5$hv~c#&PF_ROcJ0a~7Qa>i_ku;)hdm_OsP z^Bf@KR6A~!9Q2Hj!}(4j&q?9JML@<%&+_vA((4UqE1Ie>XvuAj2!~2)3Tw8MI9B1H zP}Bu!AWJ{>={U*LIdseSAfcR+38tqzegr_O%miIF_}F^D0+4iUmbO2b#G`Y&D!I#WBgx; zy07#--u{Y5WcpzLS6b)RcZNY*p2?)lI@2OzhUqfgN2K*QaUB62j!kODzYwxbSt~}c zSal?wLra3{Bvz*=xV1$gTVSYrQ~^sD!BqP|0@CVbgBig?SDL> z)&n;>KBcJXou1bwIRQIxG>{iOd*bV$ZCs56K{l2&^^M*DOzXN+`#Y5|4sGq&=SH7N zjFq$O{UEfl$e~h_?QerDn(Y(ivkg?M=Ntel37aTtV+*OmGr-1UB*g@U&7kQ^9>PxH z7I*5e1grKyu()Mj-5ZS*L{c8@547jN{ub1qsI$UEaXn zn&m(!1FV0til8s7x19FXf9Udlvhq8hI+xE@grM#!6PRCnHJ`ah zaV0B7-AbN59mvG*c1D5By5C2N=54n$X&$q^p{8JAUwteUJ| zUebuzOt;fA5Mzg>9F?#5#7A+RKPAgP2eF*diyEd(Nw4;iYS_t9){q}SLQy;A0c-gq znAHqDiOW)@5eUnwzc%;`botGQQMM7prj%Tk5rfm|lw|(^v=JhiQDd!*MSZsdhayw2 zwx@U5xkS^>1!>+I+BlA>nGIf6w*@lgs+D3#2rDa@)-{BlXT8-KFdS8LIp!o%{nZ7` z3euIbr1yt5X~o&##-(ddAEB5-dNU985u#aL(cEwtfaMz6Ap>cC9GJCBT8)(j=|0$J zgPJB}-btQjiP2ZSy5PTbANlJ-pF=d_&dr{Vf*5ByM+dBFe3S&%PL%e>N}owpGM|!E z$|s3pXpKv|kv>UKOsiqM%Q%l{qp*CMbt8z0R*kbd9L&4fL+Zz(Ej{3NAnQR@WISv# zBqJAw23hkEVUv9%!(Ib}WL(xh0caA5V>PSF$h0RxO!Cnb*1SnCL8CkL>f>Mbe_=I1 zoQt5XFpf*5&)bkDaXUaI|GPf`+o7yUoou(5kd@@TyD~CQ5u3exl}qk(DASX2S--+Z zuFOZ&bh_^<5HDsn=94x6s7`04&F=(_4K3+PRuM*TVxrP7f_V z>jx0_M?H0ZxIZ*fsgda6giI=&8>-&+KguVj)6rwkRRC6<4^W>IAi_<+0C{96lu0~M z%(G*o_V1qzRvI+eOrhjG&H%DPf{|D`6Efgz5L*B-vQ)BHWdx#|T{YY%R+l{O;*Ujv zLcue>;=@tn0oGvbHd?2AB36Uy-69`kfWm6o)@D@RN3sPop3C(>#=Ubk(HWukN-(Qm zJTE&>fHIXuJ7f7|6mgGH&=jRwj=vScB+GgzC%aFAHdTnSgYI|-WXMOpLHo@6_ke`G z0OJMwS!(ddOu(>R0T}Nxpz9w6wVLZnIWE6E?sH?=m|<8S{FINvYKx^ppQi!na11W8 z;Ta1jgR~UpHYR2B&VY5y^B~4~E>cWM@0Xxa!fYTbUh#ji9WKby1<+RGynL>NwQs-- zuYF^;_koO2hQ>rzcKe_)1J%Eul~x5$OS8rPuPQZ$y|conWvXB$-fbm-Nw5KP<%*wt z{CZsuRR7}t<_t-?Y=X4*>e)vIZ-Fv#r<3tG3-8E(kSyILBe|`OaX8?oS3S~Xmy#X? z+J7|GVcU6(HI*%hy6ymG5_3D9^ILeDMJ(>^EupC;UgxWK@qg)Qirlfgr*&g3|0->j z}r;t@|r71@MgdJTCKrN1f zHi6_5x-pVDM}t^VY+)FeDe9ELU@(q>;@Oi4hP{_l0M*!I&Id9^P*;|x_f=YS`q}XG zfBw_G>@^vHus>+mAD@Lb&2|TkwjLgr2enqiRs6{!lnH}ld>LH_Y2~R{@jk5wF!eh^ zKE4#f__uycp>K|}2zVq^$%yNG@=7MEyqr7{fG(ky&LwvfHujt}mfd6jB^kJWyw4|u zRd^pg3&8lQC@+`Uk3pJfSm-&giR|M(iMkZg<(>ebEOq10&mnAOv5)K{-JgZF?3le| z%e(*in^VT}4=t%S$aS9(wo0;`tt#rNRri(lKl^wYuHjJ6F%VP-a@$uPaTfMKZu%O; zWSq{+h;N~2V*IP8tnpFv>YPxJSJ#4A%@U*7j?!=X`~>8gFP9HCc$%r7iG8C_l8+?S z#v^}h0y3szYG2}f@-Lr=gVDHbY_V%82IESR1Dua{1+kJt&mnoLjgQ2sQyQE2L+yQ} zG`ABXtbuRe6Uf+O(G!=)IzUrBjKGho-K7QvVLd8C%@G?hL6NS_5Ww{2)2RLlx?4hM00=s1wD zf4Q^yyc{qH$n-t#{O7hC0$|J))N614P-s(}fF;7{Qz4DlNIY(Po3hiu&^w(l=MN)n z`MJYZIrBmQ<06wt#NjoW1qnyZ3i&`O`=5D+vp)GW0HMb@KkU=8ZGVK6*z3W$k_`5>>l z|M^rPnu1$2>FGXx-n>;_QN+ZP!dqnRgFbnzn0Kd<`#%a)8c}q;dxZj41ilMQS#i1P z>Hk1MPS@fao&};*_9&k`Pta!VFjE$0kG%k>fvA8KD>sZ@d z{|1bmz(7Yezxf0p)apb6*Rs!{{ou*@rZd_1Tn*}#D#oNgYMn>0GOO<1 zwB92)fT((Q{RzbKroIt5=nv?S;Vg^HKmFfnUBylL3%a!OaW;1HSg4@YfDsTcv*cC) z^rWB{I?mf+w~|oZlu3e(3L(KvQV!aQpmB|LTKyS@w*=9|dYfghwh&g;@O}ddB(ErH z1;!IPrnYYfV6|XO#T&3awAF%jq})1HeN8Omh%4Iv&V($N4z)w?w7-6kUHzeO^H#v3GorZ-!$ld3 zjzN5R-bc5i)CF}Oz&1vAKj~77T$`1+1}!TF^}McLs}w>d2nW*h3ZIrG6Z1gxu>fHw zBmv_#Q|0vWfF{OxQm(!l%ETC8XPa;hq$)sChFt3tM>sv;5E`MLub`kO>tRd|SO|!ypQ4xn?0KDUa}Q2J3C{M?n?e`1|0IFn>B`0Rv3urZ6M49n94 zo>_A|jT2cQdp-wkT;ocgw0`#SB0%Gs2?dARKVJ8#@F6J3ON%|N-bymA&p}MaYPZ)NZvG1(xUAv0R)hAhnrn3dv3fO-3C8SlNHh73k7ft?fy`S2 zWc6dG##QAF(AIFY{-J+ESw*-y%W2Br3RpopOsFf#7h8eCf;aIsStW<>3S=ef-rSjCr7f8C zmpTT++mu8$47mI6LD0%Y)vz+IZx3L?tt@J9mGe7*7$=b!*EpFs&*=mdj_W;T!@f`^ zrCJ=>W#E3$VO@%7tDJw3k3hGH%Yq)zp~{)<;kF8&q$a|37Wu0efN73e%FO4ibzeZn zF8t{wuHmpln2C+OaG;>gUpPMK*_1krZ5jYtl5GPG_w}IwR(@b6f`<!uPSZYMMFJZ zF`$3BbUoG6%w2e_D{btlS3i{u$2YN5lt+}++l#-@Nws(Se# zft|z0iA8KI99i$7N_J-I-Lhm9P$+vnp$O#Au^<$ii^%a;5wtwh%<3umaH7wEJPYb% z`sY+2?&D?(6~=L0%8Kfh%DnChcrE5%46@ZJ@9}zJ{49s=X^RB#3Yl_ z9e4e6&{kn{+2X3aLe}b)46byIxcqOvPZrgM!oUTd4w9oeym|}TDx}T;)w1oo9;kHF z+i4#HSV7MFn%QvJCqQ;Mq%5QS?tpLPx4>b=*eiUeq}7~_rTW+0yB>t@p)We@vXQXW zr&8^TZI!lSumDDMBieP7kHRrgQqKL|)2U=Gkad4U(=P0TcWox6ex+M>b=XG1&T0;7 zWfZEJQ?K3uTJ@*DyuBkqD~`R8JMucNrH{=dT_Jz&0&V<9^(?DecG)9L-&a<*_mS~g zWQTl!r%BvXCLHK#Tsl+IsT|tc#*H6(zxRRuU{-*;)5)2}VIVd<;Ag>>)Kz2Z;h@GV z#&zee^Z1cIVj&T$&SX>Bss_1ppwE!gU}pgu>Jd%?l8iv#pXM{uGo(1HtPjrsGTuT5 zw?M|94Q6`5T7xzpIvfDmvAFqFgES*6J!yLZ;ZQp;AM5x&QlYSqu$5&FxDbF2QR&=U zURBB(g%h^^GCb;|?a_3L9Re^Dbiz4Rx}ANGI#6THttu|6hqeZC(`8oPyu|0l_Q%%I zZN~UmdXb8DxJHpsx0SzkW1&ljsrZCochQPII1U)Os9$_Sve!eobuZ67OduR;MkJ{X zdG}2osnhEU_%~1T2u3>H%#5B4!Xn5i&8t%&Ol0b%diZomYm{8I4u)qaU_4d~=vOYs z-3N^fv(@s}41!j=ZdEV|-#6R-tcpm(;{)A`X&4IFPF`;ln{O%_K!m6v@w|^Si z^3ofybfQi3L40%A1a|rYFk?2#p~9#|kh%?3;eGWEgtaKnG{`NGpB1u9dAdVA)sydo zgo9{xo+(oe%Kbh7w32Z8h~;@uMD|+jlVXQZbv^wf5ah094gJKY;BsItqUWJaz67%x zVU|cp#VVfzRwZ=_8Tvg4q8lBNBi0ZMyK0dgBwVyu2V`7u?VHsd*a7Q(9=hP8?G|kS zfeQ_vw#lc!=uzi9Y6kxXW*Ji`Yw5V``@7GCYrc7oM5q7Z5snU3CI0vmBy4tkb66lM zVBF{y+#N4%_qUJ4+a)OG@+NIc))<@@BAMi9voBt-EqH0FQtxM7K(19VQ~~zG)l$Aa zfVECMp&lZ#Z<$A!V{|`t@{S&H*RccHucgn1mw=GbTMAe<(sfQSTOQ(2rVsNRI?_#JaVehn%Q@Htq#v&8h}usow|Msxh*tUM>Ch^$}3*26lOYZ_xDi9kklXGv(X#gTtHL6%E9c1yKy&YtCl`9K($`!4x&v@Ax|Gd*b33T zn(E3!y@0I4p}%GQpe(z9@mn4{9NMy@uTp<$IS|C8RE?#@Iq~*V%u;c;F)rT^f<|Mb zS&_jO<<^A&RyP&_m_agf z%SS+D6GUYBGJ>TsCaz|>23hFyM1=oVT%P&~%*s&bc=_>XpAVKob`xL!;Q`#V7=|AC3m_b0^fimP zJfn!^!P8=l0)In?TH!9o2GX`k+Y-b<6T6>4_HPPQ5-C1{+d)zp9V4F6u>DobT#?%e z((=cmg$GAVXv;s9jOKNBRoBWxmH6XoZ3naivf8@0LUi1HdQUKXNJdKiUW6zvg{Tr8 zd}3C-?ym6t9ev_FHV)_K@lY2qV?!k>OZWH5Ftxf(Ui=^*g^4gDYm~NDQ75KAUhd`r z)|Lzf($8nYDwPZ4Ol1Qa<7hKUkHZ1N)<2DyFXlGtJ@fr6Aub zZS0|4MLq7l{Td*OR8Qb#w-Po1Gg7Fr;-}j{;4Ph%4!2uGJs&&g4#JkNt3qUoPl!gU zP4(C5AXXA%o%6hU?s1^7A^ORKbD*?b)}g(gB5VTZp(vFamwn{qxu8~GCV5tg44BIm zvw|6v;}Uz;a?u09zn=HGIF}Dw20izJ&xMJtpa&sx>MLN>ihCZj->_h$T5eiG&;+hy zHg!q@j5+<~t5wj*NnLDr{D!azHN|PY4uPE;AvNq}nCxCyODtm&ewetJ zo7O1iTDc(2))Nky;4q^g{eSk^vb%P2?)nQ565{^msT#CqOUGo}4N%4&9_$4ft~9f< z+mfBR5yFN~%$4%mCLgzAK;Lqi`vnU7MwdhPcRCK6y^6`1g#+YJs11;ILzmuu^Afr9SdS|=CZTIj`t~W=;k2k_Y(n(lakW| zIxyuFP%BqoLgVZ^CMORAGkG&7T&q-xGgW3(ey5AUoDc(<9U*MxCK?hlvI^3kGhywT z&yHd)!l}MNwy*Z_QtT-1P}11Z(^$OT*o(`O6sWZboBvn@&ynU?AlrJ=QB4(z`EU+6 zlt>Pr3|Xc?m{gZIKu?bHar67XTMGV)PzXBP>?pZxPRwJTXcZyaQ#N0%f&~yE7M^hw7IC8u#iG?gNEm~4BO%MVTWC}qzCT5mL|DtgP@ZTYJtF|~NpSnoIU7{dowX=shFntP1irkms{ zY25Ma=&(UY|m{;a1WV*}g z^FJ5(2x{w5QJoqGGQHygIJr?t6Ej{OfefpJwnEgLg(h5^Y($vq7y>t>d=@&x?UwTq zIIz(kd0!sHGS$N)c{_kMIpH)0<3ASwObygRge#A>&!bm>ny|tb!Dxx{u|6##MGUEN z0AZ!w5d}AV8X=zIbvIs~zLHqzFN59h8Z=dK??CBT*Wp@^={ZGOU8jyw%&MMZsA9Nz zRS~O)9%snW*ZCN{Ql-nWt&Io^Rb2wCn?M|^tFv#)0XG6zXVfe6v1AUj8jFywH-TGi z$s3!b&S(QC0$PcTWj%3S_PKGyOZIo(^4|!3WWnvw#0`L_+d!VV7bF}Eih1|U-SzhYSv~YA)j;l4z{IFaqP%)aG`-(PitF=Na_>w4 zV<(=)fR~V?AM{9|PB^>-qlh(rViaSIT_N~*mXGHiH;v0?Mf^C^# zA!Weh046dXA}dJE6Q0&Biu8UG+UjOo+EFq=A(Vq&UH1$@V_5eZXyKo8L2Tus*O8LE zMxiZp+_S)8zoB>TKOfTAVAsPrX4jWJz~yxM&6ouMxG%e8s7676)&HZn09K=U27|GX85YROhPY{uUr6 z1AU4|UMTa4>{ud@UONNnyxU8TXho0~*Ow4S?dD^$UUN5GH@60|TJg*cqgAw>2h>s? zNL>d2s}+x?Ch*7(GQWNU85f-5t-{_xCxe#VU{`3Rs`8MI5~r5nX>U zu*pU5s%6|{e)b_gKX<-B(>|_H*pg@{!rQe6h%v9%>uRN-fN`OZr|Z&qK)Hw1w@lqb z->mQ%v53>zX{92Tksj6sQta!aV5!t9PdEm|8VU!B>yGvRGlZ*Cz&|H?B$IWXxfh=X z!eWcXNFc4wfS?)Fd^29dRvL+D^SZ*m26VPieQ|QYMG(ua%A@ah-IlZE(Qd) zwTf9L0lS_~Gkl^13sa0>O=g0ap!D>L>rhVulxmA*zvolXVSAyjddBk6LLl0g4ZFl& zwXoV)hb#)iC{^b-eKNP8rkM|X3n*N|gl{E)PEpjjR_9>nzcT9`FsqwgNtQYP^RZG9 zuEFY!?Pc$S8E5P|-PZBK4}jX0?TSxfByM-L%oHaJ^q+77=qA&b`s{)t3ap<)nDB+i z^f_4fxWsFUSt`!mZdeJ$NK=nf(JF$zHF`-zX%nOFR|4txl?S+s#13ST0+v;(CXLc3eVoj_@C_J(*6q%<-xH?rnxq`LmY@~Jtq8`kL5+a3F3FGU ziDB&FXb{K^KSNvL>9pMSr%z9BL33Je0kFXe3mgZ`YOIaR9)ANHllX+FFX&$X_>`)0 zfsEVgfv8(+22J-Yl^o47alE9oRf~MN^{wZR&B2U+#)O{Tq{&VI)RV=jEZK#y70M&H zY8}0ESCCRa<@GOjxc;xU@AF{Wi0g=Zo9~PPptc6r_fTvpNiR9( z2=G$dl@H(+MkC<79AvQqei))Q=-;zPf|(R~giu{V4>=0NG?F`!6$AbMTtLfF&R55R z&`R8OEI*N8IEK}7GCJ7Db@YC zguK-=`L{|7s}JVFBw?!>&(6wGHISB*yVqY(+Hz6@ryjIUX@u;GL?E|hiDlKdj~q13 z`D8WDvo0ljDP#?xwj7F{SOlQ~V3{ttR;f^BSO9cufHX!JKY3gC68|?gd)e+%NXsp% z8_{aHqY;Vg1I~FFIfgi;j$g$n!^CmI6=23GgENtS8S68uw0HgPALpY4xYhGA>6HM+ zSryj#TdsjLk?8KD3Uwbe0SMu$W5(qgE~#fl?`|^uMgXfxPH(m*Zt^sJuU_`M724Qz zFQ)1~v*m4IRz0(5O2zFUmaRHyG{`gsLJ1@|F66b^$sVcX0uz1u`Kch5wV4c2>ocZ# z48yazZeOUFv6!vJ-mq>aG^)>hy<(OHE0gljV+4(bRJO<}>vIK&hPVG+kNf|ax)8(B zPXbt$Y-rf6D(wbXzQXg2|1YiI3i@)c{ikEoKN_}l6$9`)_-P)rRUtI$1ai%@VAcZe zu>+FLeGbTY(-DSE?Q1?C7DRlA=?!QTmwulJLG1NrV-l9@897CPQbVvX)U&*Ki+vmCu981~gRqe-nv@g&fU;_^`!-j|BmV@W zrP%ZSqCpd$8mrSx{FiU>5Mw220=Z5hYX<1-)DQp!3ZR~zIb zC9M{9xTxuqMVGhsIdIUX`;6Nau}W~A2`1b6L|iiG8d5~gYUiUQIS!EX_VhGP>;vR4 zrNgFSDb8MRF955kdhoCq|Fkm*vaL(XYy0`sjB!uJ z)*m|D`w=+G0-l#X!&Ti-D}snW(mpL0Q3-*%-6JU2%ZY=+M8v zJXiy5b>Rj_7H>_~335U$sEJ)~#o-!;J8=puRC{N=WEfE>EGsE9^H5d=*3ShBTcwC) zVW zJpfksnruQQC~0lQ$ft`kHR^t6>xf7Rd?*|GOfr(5eXh?C>hkeNj zGr;Ksd=_x&Lj+AAyphK%tB*okv3liZ;cQ52MZ4#tt@5O$=JAm?G-zBivi9vJmpu(& zOz457cH_Cv`ZzWCy2*oO z#4tHjau6rG?FelmRo^MMIoWP!u+olD?+L4(T;9q@4BT3hb-~`df`ubgk|r`Qfm0Q= zEZuz>O?`7)FqqUAh)!!y7(L)L>h8S=8aGw?tYA$CXsaA9`Bd(dPCf$eBYYHQA0Hu? zWD1aOoqdFW$E0P9(%Oz}CEIl&94Z1gpv_fjxq_h%X+PEDp}GUWLX}Tc$+-tYTHE26 zmSyB#)QIrq0o@u->rNcXVW5iB=!pCuwhRmq+KuuAP&bwE6 zK%Zm_WO+3Jv=KSB$vc1C_~2mM6Iz0Zh$U)%BHICqf%r9Aj|dLLxVVSds2# zj}FE^C>D+btli|Q$7|Q6G5Z0S{f01|i^lWU| zgMh|1t9?!s@;vGXD$X5A!vN?DNi-CJ5?eQ{zwFg@#bp{CJ z^jCZ&PL5z}_yPdyusD$4w+PBts<$+G4ceGyp{^&`i{J7HcD}nZt3?~210&%-y!`eVJlnD>*TgZI$mUE zOE34SosA`s!e=1XDlxtBB<(&2uo~4Saadp&obm;b@quC3SvcgU6+Rt`Nf*0$C4eb$ zJX@5PGpaxf-3Z3-+E9#M50!{f*kk*pfxNEUGpQ9 z?sfG7@^9;Wd~B}lj@4W;`6ocD7QGigllOk{0KR}IbKi{smLqr7vPvO!-d(<0gP|h3 z1{?G{K&Z%i_Hsuf&2dm56`T1#>pVATE2w3wPwBAisA<-*)HLc{q^FagHTMvvH`WroIkN+hrNb`9T}e5j z%tzv|-FatE+8L-MA6@ed6c4cKWzDlo8vv6Jmfi_%{olnr*8$|*y#RdQ>LcwtK~qI8 z>`vL2pj9V)2v+YO%~Z6cM7RvAOWeZF9-~j|gT;JT{17l>zlt*oT6;}z5YrNUY{6XP zcI^XbG;2TnX*5y`{r3J|Zq&7Bjte5P0oKNel-AvE-o#@lDL#vl|LwpqXushq@4-{fJzzij;hY~hf>LDtt z^ixjuNUrF$iCO$?(%5O&Dx0N0~`~je^E9M*|JAKoOcOTPMekCuoAgf{nlC)jl%bdTIe2sffj3w5^q?*LoUrV1q=jgND`c z1yn8v-T*-Eyj*zKZ3IhWN0v|e(1lh`viO`4>FHn~LvIJLme2=*be3D8kd>L^2^i@- z)zj`lo%g0en-Hm{K6vDh9Iq0vFZ>VM%2pe7T7q{0 z!j@pw!*byJMx-jm>tHEq_8xJl38SN&rQt9GcS`-J0^ul&yCc>nQSql$o@+` zS>R;wuaA5d>SM|`vOWf|s&O4ECHF4(5nvMa*`k0ksdw&>#_`{Sn1Go6+&gJwe+CL0 z6cg5a8z60v>)S^j`4x&P=z^xxCc@SgxuiClf6c1{)Da9vXW!q!~3ZqLk9RP_|~&NAG7RGkLUo3Aln=c zVj74cpk9_A0Uh=L_hoC&1&#zVC5RW*^YsScOKdoOh*yT9rQXg>NHQM`l5X()M@~Qw=C}h=9=OMfv^*&PHL7>J5#wr^3uAx39 z>r7`B_~9fVlaU&b#-0jkYGRH_?7u!%%s7Z*R%GNDa+=S_o}>vR(+Dxps(#6JrxVMH z4|2H7%F_y%9;4NXGkmr+`j{K`u`@x87hP!Xr=&@OKHF1fo$YBxUL3p5g*IMwhILom zdY%VnW$I@%xSu&3fHL`BDGf1ygiqpb2dO{uDHnq=&DIp<1`S$G)x5%@Xigl&^5Z=_ znChSKIfP$f3FL<=Fk=)e4QC|PK91U<+8?BEP$1dLzK=N zyX8Qvt`F$uu?eD9BS#Spf#s5N%OwQ0OEG(VH`ZW|9fNTMt$6m& zjDl^(1DJlYSJn&NTK{_$waVe>LqEhIa;1kNQT_T1pU0a3WL%k#ft;y`=@HLG?R_)U z|M#+DayV80DJcjKf2d3N;kSU9j@97JvFELjmbtCc<#|P@9p`)B-bT=>5pf5TH%@WN)9zNVRG@Am1L^vJkB7ws%dCDF{)idAfNExRI^7eB zP+wk}+V3HPw5C37dqx=6dnMm%xYQB*4UoCBpiTZ5a5gJtMPsUPO}6}D59!xj^wRF# zvwa@)5gzb;%;$k$#NChkf1TQYtu&l)rYu*@A#C}xRn!HRjDN~QJUof|;^y)UP^j_A zQ54v0o(J^9&vCg@0n3L^eshyDE|)I=3OlZT$#36Rp-r^NFk;_bIz&NBr~QpqkQWuQ zw!}prvm|*neI3k%!m{eHXTT*2lp2-e<0Z>{ zF77KefgJV;h}BYW=W!iBE<3LBse)>D{O2fb9ow0e^d2iuPbz90@zDh?v`O%d&x^SM zyCL24JH@Or?1r^Ej0y2?0j+=Pxk2rENaHfl+k0y_`o9Y-_s`!1X%%&!3MIvy-#x^b zrdA|5`wyV7?z*CpwwnP;b?;vxPj2-QI5lA;Jmy~y(8XB!RyN(cR3kMW&{p%Bfs|Gd z*s$DlX>$1Xp!BAyqzv1^M<_jpf;O*i0cvW(Yd>%71ZAA@Z3=uJo3#Wm&e))(B&W1R z&?!8S$9M4&Fj1xChFyJxN<1m}R7Yz7IH)elLJgMGv`X(Su#HyCnkHRzM>C&yY((^) zXGS{i3u02``T>tZ@F0~!mW;y!eX??4XAoQBHC_S2?6E&^NwocX$}**`J$VOGIv)UO zS+jw58<#l;`V@S3hmQqxg|?!iygq``eR;4?L=V(QUnJQbDAdh3$G|$oOgaRNK|?=i z+PVkfP`BK36l!#wSpjIgsaKAibd;xg!K%O9FwoQ5X=T=N(8dXFfZFk6$Ag5k2ZAa; z(I?=9BqddYp;^f3*Qw;_Ax7xwr7Tv$+EEX;3i4+ofK3wM-l4>Rd$!kd@yQnE7(xC$ zjj$;U#=2k6fHL{jC!%yDuHX&>GDUQs^T7EOoj>wS(6ArTeuK|}wDR=@d0Yq12QXPM z(&gp0;hxqjbaM9yXcHh_XIKJsiLw4-KodyN5Z7rUAMw$c>G8J>sH9W3r zM7TVnv)I5Waf~7K8d;_lutaJ*#7H!)5(J5GnM>wsIBbd_uOARtq=;3Jxl^u7LZbFP zdrGmEplK$3ITM7Ze$೟ijeo>0}xRnJNf|R4JFfvQn>!Mj$Guwf$l$b7R)0Zj0$ zSutVjrmpQ6K;wbol$8PNqW2ZE+mLW>FCjT=R@W=|!|I^#bgB&`KMsg0>TYg=hK*qa zmy_48gbwS+kuc^74g;?OGZ7{-IDls}F6SMu0k*ou@e;w7euW~X+V|@%S6%C4(A~MT zB5fytSbf|k(=F47-vDT}#xokz%M%J%Yeu6nxo9G!@x+|3cUt%W>CIqPl)gLRk<;SnEId+e~sAT2Yu4?uedKH;IjJr9M8fI?PV_e`Z4 zDP}+EF=r^${`lcj9xL4tl6fzHg(HhwF*zk@_adkXlr=2F>iGEp#t55)sD3@cK}e;d zi`?-F|5`KYj*1bT(ms0?&;-M}rd9^N1x>YiDo*~@u!*slaTcX(-}8u$m9YjXTMR<} z(V|RSLeTOry~u)x=_eixidDGf>e^+Ig2uRes#)Io!UHTL*-=hk31BsGPePFBnJ{^{WDiX!5U6a-)JyYJ*MVrdHNdwYYd&d^~gc?{tnFe zN7dO#=`b`-L5rxXemv=Uk$4T5aZ<2%5S~!Pl5(b!#E7p9H-a44*(GUtPeJ1(U#!AW zal#KiGkbzcdj8^(m>zZTG2=#rw`=s_g;O>Vhh|D*`MYc*VQa#AwZzr@3T-Uo+Kwdz z%R#5#eH=b~oRD=&n|u+aI|18qGZ0y(IMaAe!Q)e%f*UTA|0QUm;yONkS*Nq5+w4=?Nmt{t!S02pJyCcfNDpRQ^RWGWcrMveFgI@hXcdyzunpmJ?4RWCGDu?CMh>ja$rvw3@D>7M})awZb~XxmR4yIvvPLR5yk|ZdSnJu+4Bb zqSqSZa3zDc+Bcj<9OEh*b6@v~%UkDwS^1PEr=1Jw1=>r_9S&{fnA7?b3K)Z>H(D6u zF7P1F@-x8dba%oCKpo)gWa5PcE!UdJ=+Sa_1R7B=$H`I+TZVcdYzFX7Q4hLTmo-b7 z83VLNi`qDQQ5A?)+I{_%F4?vg$hsug`_(e04hR#|pcKdsE{fRQ!VT37ySxa%>U%tB(k}7ctRrb1!u$4Gk_3@_5 zp^a1hb}GguGHqn^hQ=k5kH6 z#m}6NIn06%B@lkD=YdCn!fuE8G$Ma0V0k(R)j;-s%;!?8A6JPhZ87++rq*_?A|^)N z&y1M^88$Z0%Hii6$keBOCRn5zWYqJ}#tKifxGD3C1wIO!{|Z^5G)uBzX?W%Jdm!zioT#6>hHyA| z*h|l}P|F_xt+se->wBD$wIE0rHR-XQpcSc;^AaVkNPL!9It6mh1|aKhdK;aOw%XA0 zqT-?Cxpc~rD-W4%1hb;_dk-LwD`FYy;!LH#wh@ZNv(Aut$L~H}6}=y(x7iG0qQO5S zA=m%y^Wl;=4=u;#?WX&dqOdTmlD&6;wyGp~E@?uU|C_f32gu|dJ&k)d-y>8y6iYVE z2*yVQd<^FoGqELf*uKT^t)?$pfms7`OZ2N(kyDm6+ov<;Up4j=@7+zgez<%B7&R~NaeH^Hz! z(loewOzI0_MHMsJBQx@?Le{)RoJx1<4{7Y{2PasY915W9vsF2aU}?a#w`6gQJ=*^7 z*-y4R-luKAYlQofg8<-*!wYG3g8x$&{5+E}7@(xoDK+5q9s*+AagyU6=DB$&kS-{R zAb%)ftnwilNuT7S>-tic3MZTlWO+uj?!!GZ6fvQpuKi@wX+Du22kN4_^%+2>dtB0+ ztCTUpOskIK^4Bn+P^I0o`O@S}AIS`dIHsKqWP-yfo3XFcIRG}z>g{oT;Z9k6`+Q(y zC`tL!YlKh7YL;{hF9a|tGx@QR)9K*mivW!)_5ah^uG7Up#+5te4x~-gXT{bKQ_g`3 zSSoH`QI|lbDHNts=XGwuRDmF3+=WKUze>>B+>9Pl?PJ>7P99JsT%K{gQ168xo#r*5 zR-yVV9~rnv>9F%UkE3q20IHFF{b6BcTi`Kw5Qp^Flvs*tQPD##MDBz<6vY%Ci*@E6zrbz4RJI!k)@}iJwos z2iPg_-pk-o05(!#8@Jb_ij7$K(Yiq1YapJ}^Q?5ajIfo1Qu5f7RF1WO)wSza4V$pg zc)TdaLt6iO6((I<2JBDJL5dovhf9?(S&yoI1wAr^i5 zgwFK}hQqFX5j&6<{)6Gr;HHGfZnJ#qveJGMLF1OCT1u8R(zQwYcgHFGPrY>}Nom-) z#(2x=#hR%A)>0{2>a{!l{|pB-`SN=j0ll&(zbRm??cO(n=N9(@8PB1wZ6GtIgN1Yb zp)zm=lqzgmsx)X~!d9wJ8vdh5CN*s}BIfYR|%pT06Cb&o?C4{CM9qI>lm5UWDCr)iK!6bt35{z&CdL0hRg zJfsWqj{;Vbe;ZLwe+JOx6IyYk$6O$706txM_B_JoBH&*XVd2I|F-!kJM*X_T%IAC@ zjBffii0u8m{hKXH@+~!6gQkP`~{PIi0uqyNEJeG8_RgqB1 zb<@^!k&oeQl7XE28i-Y;G>^iy!5Wa_1+j8i*y^Q(Dk=LA%qoe~4MwxROMEhXWiwn- z9=g;=;ge)oFHiXBKT%vaJ^B+6Yc#BPT;b5HM}O+0MN(0N$fZthAEUG4ZEfDmj$f)}}eQ-YlID|1(`C5-m)#h$-3XvuH^;b5*odj1Gx zO`eOZBXkc4?wT;iQK-6L43AC)&^u^4mV$~_ZbDqMb7*gI_&0sVZ~R;@4E@e#KSoTGsEt`djR_rBUH!D0LFVP%eig+U(kp^jgnIF zkB^;GOO9+-pmY@FkBiv7^lRGL@u8=!FKVA0*{?XYw1?m*!XNe{nKNo!M1m-KDwmaRd;&Z=+AsJuTC zFk^!^%~+r8;p4D~;$8+zt;HR|{*S7&4zRPh+BlFxaqU493M5zvkd238En3_up547S zyPH+^?k3sbS{#B?tdy4E?!lb^!GpWIgy8c1o_XFom+!AT=ehHaoH=7>&YW?=<(|fP zMkK}J>mODk=wh)YW89p)GJt!NJZ9^J6+2sjg%mf&cY~m4FVq2}SpD?G z>1zNXAQhze>mh`*GypueHUN2Z(6YojwNK^78-|S%l>2g_X6!!{(CeR)-o@29Mkx*w znoZT%74Z7Gw#%mrcY#=DVV+Woesw@*WHOIJ={^iZ4Kp^j9G?H=N|>xLCkvtcj3#l` z2*TzZXX5e6s7#J$#64&Xr1Ro*ii1+Nr(=O!1C*M{K{yJ-^#IKU22;2&VSNBM8h6^e zW7!RJgt82sV#Z%HkZzB}Mwq+8iXcO5M=-UPK`}Jk!o(bmJD) zzlV;Av$rw9+HSL!}0LWU-`Qpyq zAlzETI$EVW{~w?(P#oM0#2>)MAvG+a@&cjwdJpx>)R^6ulUUVV(>6uknz0Q6a zGaB(VmpnJeXK1FBJNR1azL#4JSQ%|{DTvv|JtCa4iFReIm}GZ$IS>VFr3Bk4 zOlryf+zMAfxB}eFjo~6)y2dt~B*iXQIx_1Ho_-VIe~~t(*Y>LbBpgY(bYVd;2DbOc zMOX7*AsOY#8iB7DVtLduFFoYPxBb4I(T9ip?hcx&`qPTpFT9xP)?S_y;YYods+zjOUj*Zq^A)F;U{@CX> zNJ#=OIp0n&PsPl7JaMnl0rr;rQ4FP zfw-ovY_ipH`2O3%ZVgK zL$cS|QlN7h-;3eg)n7rogsr&V5=#ev4J0k#gcn~DeQ&fMq%zV$vz$fMY_j<^9-uy#QlYGtk@6Q<5Oh8urCIf}I~Dh{MxM=t?Dp`^%}e<{MQSur81PsP%pRT;2F{BeSoTX0DR zqW`afz>k&wsrVa`FKyFcMVWa`^HV&MUq#^9nHo3E5$ z)XLCqC!QwLiQEB$atzeWt+Csx&@Lzn7;f9+!9x(4bA3E!Dr`QcC$M}W(Hg3>wFS656;?3YiY%lBEmZ8d6*NLA zKmFa?5-y~=4Fy3@EVCVmQ{(D5vhw3D0PdB#E{Gw7+W6_+ax`6vY>oT&04SutMeknV z&|z;NCr(nj)^X&1AVsCM+L|renN?5{lwzrIIYNauM50S{p~EViQfQ@fNIh`Tq8!n} zPt2(sGS*d(mn@k)eAft8tsd$FE>n~<($X=Zm+uBQWrVpxj$2 z_1mNzL*A2E2wXTBq$3a4fK@qJ)s3{F`w4OZE(J~;r`rR4&5OFQ?CZa*+X8t65o zZgmh46-+lxPd^yKZIDg|;&K6Q@bL^%#TIM$A)xdIG$8vb=u8TUh_?>|a9Vr3v-%;eVluE+m^T#t6y4 zwwL73xcICQX_09l9$a`}f-RK>yR`&5L)PNyiOC%WQ3)9nj=YR8lEmGTrr1D)xzFW= zI7ei`S@aK=LzzV=U53SDqDh6NdAy@AvW4HtH?J@(H<6>SBJ7&!`VK|B;c5`G#6GU* z9fPj}%LxyUmv4l2DTXnMuXhuqtP;8Rok=q@{y=crgov_(OFJL=^kj3Hxo%u)%&^M})4ZVHw>K**+w7LF??s?ri zYXC;>nC0IE@VmxjL31yNi|9A~@SlAjP%$3ZGaGh0_h+nL_u=Fs^F?9iOu9C<>jOC{ zRzR?zFMfPDLmj?~-Pv$H|&a*)mUgt<&i&3O3~jR0vG9<47zm_Nx#B(8c1K)h#`eOzI)2K`uxCEA%U zgE)PLZRw^;G3He;w@UTyK_!k7=ybbyPO3Ae)c|%SH1cAv5knF{DoWMa-0r z`V5N7Vq*7F5p3#>{l5Tp*4$n1X-?O!&JbA58vB^Vzk+tlOVyl)5(;pPU`K0Cic4dWfcYUC9`WOy6@iM?qMod0KLYxd zP!|^8Vx957m2xZ}J10HbS}TKA%c*0G?FE=4N$=hfE3E=TEt|VzO9hJ-q6u}mu@d_W z^cQV?EHrqRdmxC*nBGldx>;;cj?~qOlQ6tptO`;cNtg@Osa9Pr$6|igOK-cbnGsH# zRj_A+M=Crs@yEqq)- zew=> z+X0xue8Jt!dTid^{%9U$TE57SHck`%)_-WjAgoFxQkty3R!w z-WkL-9Xm9x+9gvRz448xqQ}uEQK;kbdbf;e{hF9@r!X_aZwP2)I$-x4k=iD|5wY%G zU@juAbep1Ev@4R%2HdHC6pC=s4`Z&q37bFid4^5BfU3yQPswTieH|Zxk8j2imdMG` z)XEJIh1AMmX#>$nDiwWGLE^Cl?28K;b36uqv@GMH%@fWPuATrzad=xXerhFbdBmqj z{H?wCtCM$S6gF!ydXuTze>!Ujml7C2pA_N(;S~lSbzPaBz)~#k*&n(P5{*N}s(054 za4|eF^LJJ{@#P%n_JF!9vg=%GBBUFOONBbrY>c-jffZvDBW80PI0b-eCzap32jqX# z(mBQ+1mJpet8NcbRne45@lS{NV@_>2$2=SYUbak$-xMsm2R=MDT zYH{D0&@P6Z|5W0LvwEXl#8kl$$*{TW^mmrh&j?t@mQTOY`5!Ni3S4%RpRgy6N!P=yGVv zAm`-mSLE0g-Rr^*`l^h``#)uVNJ!BE&_JmJta>$&<%9dR)m`&Qh$tARQFRm1u7cFv+z0;q^*IL5PqxLF8=+~A5o2Prn+TdsJfX^cjvWNJJUr@+ zlvQH=n}N(~E~&5=r_ui|+X1LAYN$A5dXA6KB)%Pe2auCwDC~>ZL_10L>1iCB_f8;}o1Ry0 zl0(wR?*=1PDIoW_m$3P)(UX%&HOOc8ffiGN&bQMsfBpS{MJsV4sy@*t$dwk4r#=Ab z@?cff7Slx+OyHP7wU2lX z)E$s9beCCBh2ar}Pph|d=yBLmJLVN@Ws(7~_PzCdQs zyS)bBv}r_aS5VKTn9ID*AFc*^fx37|wDazM%-0qu4I- zES@mO*JG9|z6I&@xjfh!i@uXTLCvfkL@C~`VT}K+JVd}dQ}5<@seyDn(k;v~P#kc? zY3~EUV^*JW=Ldw%&91H(_8}BK9k192e@w7i#}T7q=%>&wBDV~p>odn3IwI~;u&696 z_tq~UNnLhgzkW$LCxS`KSJ1AzY=IcPD$(@~nA$)m825Y&K>@h&xc+wp-TGSY*Q38c zdpU{vFaVmxJ4Ko zWxaL4`~-`#ac^J4!BjyWy%;gtYy|PZLV(5Cs>?fU2LYV7yo$!*HJTcXijPFQsn8H5 zeG9`%+ftx3O`4>7m(EF`^+1ch3^Y>0dp&XUviUz=;+Yg@F9&Ub;#eWR5a}}3%hO9T zmbjdA%vg60P<3%GFJ9^VOa z))TmoK5KPIbzEh!{~8cQSLux>)`D`Yp%h1XPnOFLb#xR%!();N@^4^NTda;S`A_MK zUq%otn9G55U&KiN#|7${3YyRGbxbTi2HG{J1E5;bp!=;rm%6(w1q_?qO@Lf^U6w+K zcit4ls~4W&Nq)4`^Flugrw%PsRNmp6J&q9OR+9sjV?=r#Ep?Pm1EiY2QLuAkHcA8eTeT16UiI3Y zbK3a3@*MZ?-N1?hsW&h2kDQ8xl~FA26Xd$VMph_tn7{cRfMyXVY7AiU=3XE!R!c`S z&h3~^m}@WD8`w3(j-DFsun&NHQ}J9L+nLyMUr$`!$vcTda|&6lHwMngA2 zXM)XMa<~153Q#dr>Jdnt^{v+fmPV4cn5wWdYwD%_V%v!TZUv4fBr9z~d=udR4evCB z{xMS~X~szdw~1Yjg2wo0D>;A#vkQ%o*<`)S1}odqbB&_ zkQ~p(Y~}dhp+K4Q>=ds!0-)G@P!vz*2BC*+iy(99n% z5^mrjaKOm~i)NRf!I)>N|EqMxISM+R_7*H~@I7@Z2yMvw8*#P5F0I~YZaB?aMqUV1@Iv<#Dsib0 z>M(pPXHq$SBi&VpFWX)QWY)HIvOQh*@=Rj~Scw+V&LSPm(wzrf0pvn!UKkR$2yh1& zv0jY1GJnC|8G9%!%&r1*ChQUG;tkOhi3QrLR}*x*B!zcca}2u%&{Dw@wcZpid2LQZ z%bfT^v}}*FIp!jzY{1=?d_OaCBQz!~h*o_J%*nHpVSDyCw7UqNS)%I+NOO&)D%X_d ztKma|P7Ojq1U^zXu}Ba>c@3qc&|AXESgpOw7lg^U zp&@Qouo&g^4l3Mzt@xKq!tW{Q7LvNhR^A)U&R{yf!p~AEzMTVT_UKfsK0f#!fc!dX zmZN{jU(lv)C0*{1w*{L2*k_mHkH12@B+2}aMECg(sAx`Dvf$jSS4)%a?xItmImd2A zkM=$+#N%Tzut=`@%>(M>QFoPEQ1oshsOW6Ixy<71+kSxNX*vqjw@>{6%tb|&-xdqa zpJ`SWt#Qu+nbvDQvDbpo7RuI1vCKkHMJqLPpI9jlUKkAS=vAwxMF`Ux!|USo#R+<_ zDO`W29oKDw)pqoU#}pQSPU##qrirGs zTywc<06}L`nple2BB_&>F`KMJ*vYfxW`)szWdIjEc_`JIf1y=!q#iV~tucEbK<l;U8jWFHQ=BQ+U zIj&@DzKtc<1!90o)mnEbgfl`}8>iPnx;?NYX=>(TJ_Rnt^}?MreH)usr1#3<`BRZ! zi8Kd%(=HYq1p>=3eWyd5Evn#9588H|{EP)r)3J(LY(2sf-qG2rG-zJ2*wP1WSbekNTIj}WG zYzOVcW`2)| zo^d%glU%yawOT!p8In#P+Tx5x5Z6LtD1|X?Ij9Wk`l1`-7BdUx+iMu(uBwX@%keo2 zW?($1Vu&orSy+7#Khz?@hGZ2utBFW(Ni)gm%??kYHe*@|n?pQ%#85lA&3|(K9$T~% zcKvkx5ly0t)|KHc?>csZ!iae3{`oUp-F3wM-Ov&N)-hizT+ABse#=^F@g5*a z*YFsrVCs>KS))W0xp4rDfb1-!D*qkPERuBF*%>|9>n~wSN7PM(D9Wv;(0&kI3|S*$ zU~?cScp z?2f4KnA+-G+e zyNE2hB@^tk=~x97k8ic9Pir*h*L83Hh#S?ifmr)Ke>7@r zyr!UeElUBN(4TxiklU!Etb>0T*70DDhJ|R-?A}p>CUwgfeJS>Qh*+)!izX6Nzq?qV zMAVLBbZ#mE3Z*!!7SFFwX$-sRcVMAoXNYY>wj1oK>{8Vdwn@=t5OJXi|mpTx(QQvy}-=2m9yD4NPh> z03I|CVG2pttXHrY?Jy;X2SgW5H<=A>GKX074}j*ypt|^EUMNyhb>}GTewlh|{3w92 zk;Nb0YUYD*V%bwg93#*TkRBQ3IADRC2fpSwHPJ0g$KuIvg84(LEk^l8o*yVo*aCC*wU;)k(U0`e=5*D>gWFJq^ zwcuh2hRwP~5laEO785WyVsW?EpFmt9ox=@@dzJ<;*GQ#dStz)stM22L%TalEjG^LY z0ZvWt%poSHEe~Qb(IXa;)A?7(kz|O%HRnf#c<@gSdwqOfiG^1LcTsc_!`=EC(12C} zH@*VQQl1~>fCCBX`7;>hW!cT*=KGbPT!3U7&3N(5%0LApbPS(tz|USK$Ezf#oy~E} zKp=R>q0{z*9OUTwc?B(L6Lk9jrfAohi+H{8v~4vISj56LUQ^h4p?;r)Y8Mu7ygHyu zjeSU3Va0F4%w-NdHe3UWMrT@UjRQou3OHirc5x-{734y20@55Wi>9c(_0cgT$CkqZ zO@KEFA~ktvjyWQ#a!a@1^7K0Xc+7~Hrl9}CnhDeHctS`apggnM5nHbd;Ks`8F*WPW z!d*&@)av=42(m0PAH>+9kWO5@V7z}oh!bbvMEa)<18}`Ktm=uIMVnJxz?B6EJG)vO z{0z0m=n=&6@S)>vjZQSelB04;aD3tGzoNx z)5#l70mt}@Qg=*JP;!>$y1`?iimG($Tzaek^CS5RrZbe+19I)SmDLod`~@08;a!Z_ zW&QjgT8g@OPz{#w;pK>9VVQSYaQCrn_`$9!U*e+M0ig z0gqcrZ6e_iO~PpSswfw=t&?7`#kSC9ZiPog=*{m5p*eUIWSQ;qpR5PvYH@iXPP?0T z@7JkCz@J2gwa^YbEP{jJ#s0&hqg-Ilh~<6i1B%GE>tog8f;X*8m2;aJ?3vDr^R$vSD)irV+pfLu@pq zsqN(o0rI#=i9hJ5x~>zyl%0gM89O&QjGb;gFoCeMOP6}p`JNSK-s;fo6EID;T62B2_cIiPjp+{WxwW{B_>WAo^rU%$7@jGK|VSUA(r{ao>5(8*~3B1G2X|n#2KQg{g|2%)Zh~cI?1}RC`B7i%-;^nanvhb=do~C~!F2CF z1s&hUB%JFCq8-|q5Pv%-e~ax4*6wk~xgd0u^uS)+d>)i*g8nGBKOYj7u{InT%Uu9L zI(h~9fqxS=Gf;c9#?=>Rx}nU-_~U<}U0dI3962pV!0e?GGcPx-rL%0vUbzCAoYGu( z+*J^+gzs?iR^c^3mNCZOG*iop`}J#qs}h3IeI-u34#YXr1q9p(K+5!o=D6x+2-i~_^$2}o2XM(98Rc=# zHg3*I%HOyX)YU8Xb+E(ii_PxIkS_Ua$ZZ$osxdXlCT0D5b0j^1qDaf$mm}dVLweo$ z9|W;jRi6;RqV%EsC2Okm5Xwc5<}aI>s7Y(=3=r3JLNAW4_Ymy@a0;zm-vvU4B+_{WL1UbtF>_6j+C-dhx_hztP z`lyqkOrAq82z}1z5 zusB_`Mo1R&OTR$a0+bXFcJ&pbyt(YmJ{Z-?`N5nY_g15;KYIZ>|V3gycH~7a@(jb25n~<)<#7>>FUGp{oOkma2 z#`!aHv(7spWY^UkYrdCb;{<}D9Pt4_ZJy%ZRcGw@p;N%`4DX{Ug_ni64wONU6USGA zNOV$5jQlvK;oGt_(e|H!xmXq4FZRTMPXWx@x zX}_Gz1Vr!(k;OWL8`muRsqZR(05k`1*e2oLPe|2jofCS72ZbiuL~Eg5dw?VR8F{`Cl2R=yJuNMS)x(ahD=C2=NHO zu3QIuNqh$`R{OR%KD=0v8I=|^tVKD59=#MO@}M^;7#SX2I)CTa>DVDG=NR&+gdNRC z%R{=sxwDGp%L@7LIxJ=z20&7NX2a1d=6{B9zjJSq{4;WRoT8waB|poZ@y1F3Rlm`^ z6U1S0gCC`SZDsy(cAV=r$2+3^DYux=Vb!eyP)u*d92Hv&BP(1ZEjNf@A+j8D>SST? zYWa&wSr@;NhP8wtkaY;;)HU+|v|L}3r{xBNx!cH;{xFd)Rg?V9KPlSn!T>Kr?PJ!; zF&gDf;kO~sMU`>2fyKZ&If5?c$B=cQ3xnC@09IxjX@B^!{&C`Z7M^dpSgOdSDYK4{+40CmEs3?ZPr`%@oZjs~(cY0N5|7R2piYss-Q zVvX^$5IBt4#wC9t=$^yYP#0wIg168H`3ss;KJQi*;%FF@^G1>;(u)MTRGO}({%GG2 z$gJh1z1G<3Z<%h8)yf|>`Q0b7IWNchg8WH3I{H?$877CNH2I;M<&WACbeXKYEr=H9 zOf@#yf-toiJ1Tk=EV7q}`01jZ7GsM(T<-5V29NiZ;gqGKlk-xL=q*U~<< zxP`tQ7HqmNkSoKrQf79}J>mv|{*c#DQ8w&74!}AuyD}`C5)ZG>Uo+}4i%1y93#2(< zakMB*jWIjfrGcO;Ak_;Oa_5vlJc>h2Ekn6*tS`C7(h-Bl1GzmalUjPPOh`TW5Wy}p zx_IAUi&4#hE^|ZT=l2r;%*6Ecl8W|^Rxm^Y+vN1<`sJei5t7Ub=>2v8H`wr@L*j%E zNSC>Z{)+7H-UZ^k8Mc#rRN`%c)Dm}YPxcUWigLn+{qH0IxWRqp*tiI5RC!!sSDuHj zm!AUVd|8Lb8wWya`J-X+ClQ4-a3vAL5Il$+3??2k#a^qh|EK-1RHh#-b|{#c$ucrM z0rl4-fLs?wN}fcBTaR=iGVnO{pM(n;M^(S%asG=fMr^FG|KemQP88{+Thg<_xFbFu z$d%?kC)%<}Cjz)gJST2l^_xH;HsoGTbV~4+5xZBQK|a zb_!uv0dZqn9+#d9BF=Zlk*7JL?n4bei?F*sh6r6J zxwLa1?iE%H2e>A=6^at^ zJVNYyTmHPUk1H(kvH;gXTQ&M2jkd;gK$i;3k`~khlZCic+%<$5OWy@Rze2?_Caw|T zHlxvb-lIm>jjiMVyZK*zj1}Jt_Yij3FgsZHUMOd;9>+d|?5`A9496NqlU_5v5o*q% z7l?K4&*`w^eDnb*mmnPk&VCfSnl47g>0;x9GXOpI7DoV9xNr0rIK^wh(dRn~BW}D% zwA$kYUB%HcQ-l*_R_bKvU~KqBpxam0cxYbZgqc8v)Nq>xm#k0b4~tCMT750hvBuZS zvDK8P0L+!1mN@tsC|9)${r}&eg`|A-lyAdXgqn!&MA2qlS4;fxAyjp-#uk~uj(tlU{1LF5ucAxLNz8v1XgSltLoa(o zHnJ;zlB2SDYwMFrHU@kK=z3Y_S&4yy%u`-@X8n7U02tGS4a%1UDHDzH`+GUOWK6of7OAa_Sls-RzaKd|W+-S*GJ?}u7ySYN zE3~ql@oP>DSLQ4?)tnpu21p4hU`zaIp21aSReF=P%nTte0P9L!-lijN-4Du3@sFROudA9LzlD+#r4y z>Bbx{BUxNtE&*a`((813C}LMZmST?H`!5Ama5Oz3v6t7UwUvvFNU*EJch}o+>X}Zz8$|_(qlQvgzmx(us^#Wlf-dzj8RUXE&^{{mzsq&QGXdOz>S*ImeTih(fA2TvC_&;0+;K7H3F3i`F z2V80`KLXfk_MnJ7Z)E8p3ZQUB3(Zjb)X=8s}_Np z#AWLffnv2b$Cev8$a~AgaaH_ZYt= zVM*1P7`hcf_gi+P9g{e`EXOo~#6^!BqqcUqwR2(&-JY;>sbeoWYX?Z`j<4wEn7AWE zktGTa^|?Vi0Xc1%w#tkmt)xpg-Xmwg!ep>y_tbZYpu(iL@f(83(08CA(a zdqKKTEMKcxKD&30BoD7E?~^0Rr59aFy8Jo&f|?l>+5RC2kL?F!iA#NRk#W$p0n&MT zeU6P13d_uLd??7}$BQVh97cB|h?&8DVZ{3f4`Fpiq99N@J`Zve!6KenCXFGFUCQcVpL}F8cmL=|- z1Oy+)4vU#ZuoLfnlR-T!uq{G`#05{<<%lW36p}4m8n<2-T};LZK}DB}OEKes{IL$G z<@-iQz;_=4T=X%HNVhl)k_yU0-hm3cT#RXQa}@&)2XdWZs2-K+65>cr_1GXYQ5+)3 zOk?KfG_e*19RvHDN}O;cQPfw5#ViGtBlpr?RM_%c-9*IC!dyT!!m;AfkWL(@BHX(= zHh)qzA&bev9H*p*f_@e4GO~0|OT*?9fZWp3Voi>Jo|pliu)|N$uEludR87gDi#~Ou;d&m6GaoKqtqVFl_>sJR^UG4O2WX5@wg1V}-K_S~Tq0 zGo5H%cs3}#h&|ZQ_(4QbA5M34zvUM}6i)`%)6Q|2#b?~Cu!Y=J;#ezi*~)amplFH{BNa`MeNttK*QV~S>`Qs9Yof+aN5k9&=hsZ z^}y~{Qm28OTtnz*=+xptA(A-kMp5%h+jBE2$oD#fceg1F4s zOZCO-H$xYzEG=tg&QveVanSql%86(vEc<)C2Xx3SK(qkHHnI7wge^-r43X@WI82@@ z)CucIOouUXzA(PRTs%fhCv0Z7qlP&74oIwm2gO=<7Lm{rd{Tt#@t3geUEr=u(o1y2 zmqJ`F3=h#<_QgN$0kXJMy)af{dJXJsz`qcNIxNNo`r~rPgyekUg?oW47hN@%g_T(0 zK5*yEk#IfL|KULpR}NM)fWo!49|BY5m|S0fgfMNS-35LWW|)$us4qSeMT=pQa{prl zU0T_S@Z=tyX_LnRi`-ZTkn(&_l|7%0lv zv!C%4R8{Qt-Z!b4JMdlH6n}mL(iP>_h@U!|_Ga<<@EG+D z!KyeRO^o%O@BR+bQl%2lz6a#`vPbWWYd?f`eJlIN>!Oknw8y<45q8~WoVwn}ko5nw zk@@0N2p5GN1!i<~?7O~l1hhdNJX{0e!qD8ESRutfW&@Ek`e7Xp#ZjUiWfJBklAVjb z0kS}8{J;|d*GOXypjnFN4~DYqzXfpS4LtJLtV{Dx2y&5H)TQK!~t01*aPg)%F69hc6$J!r6S-^Q0j*Aa5>t~Rn(a}Zp z#oINYaf;q(xae166kNxeD|Vb~txQYF*(%z_$GkLc&wB+q8$Dh?q|byn8!pyR{%>nQ zQpLwoa}$FKjAdgu4?!~tgBAKFEzr{sOhY)&y~nYC$Pw_-i**3P`1pKaE@rtMo6cFH zb5+x|(Ld&|FgW3YD5{uVVTFn*JzmTa3^VNE^8;J5+8Ux~0VpS(cI3~CM$+VKV$6d1 zKO8gQi(WMUv8s$`79w0spd3Fie4|jfcriei3p-_L>sMSnCxHf`Csq|rRapv*jL{-I z8c6NTE7KCTmL))|^35Z)OF}v$od7EuPpt_w7pt#i#P&;NOnwOZl-9<70&{kw;x`d) z%r3?NvFNh^a?@dCNzWX#G=OV>F^djAeil+lFl$oX>YryBAexhT@{ncozg^m@A!NHP z59G4R%Pl+oO8l|Ezvyj<%@uSGr7{`_Znq5paUqi9fOuYjv#ctBeq1p_n1iOpUj89?vQm%}<{FReFyC7>N0NgVuF1!Q)pBad$*lJj9YW0m zty;=)VGU?+;lsE_fb%OnxW+fD13ABRD9G978X#0&zIkH#H6dIpEVtqzk)sp z84Tct(&<52mW;dn4rA?=-tX8~7>VjpRU8fERAjX|goj=MT!{3(PhWg2#QZ{G)GO^b zBk!F1_Ab8&^n40~(vA1D5g0lQ+zc3x24CX+>u?}fiLEw0 zbb}E9nzcs9Ulnw841TeEV#PWNsNk1u<9Q5Qh*>1pD7%k_ba~RzfegmuLV+$ghSsc9 z&_3-n1`L)oqkoGbf5{Q5PTynE^}#%>Fl^$+dV@@J(lRzattdP^b5Xl5$@GA~1CgtiVC!v7xL}yChazbE3Up4G0gu7z zKc}IV*Hk%EkfU%6#G8xTLDO!${MfmDj*UiGSEl0Goq;@J<6d~#U7%bwR8$&Ne-`4p zbBf3lPLM7XD|_s5wBUbEpug&F?(1o8k5hLCsS1c*59^6D1W`5ZPd`)GAC$Ce?uUUFO$ZGHHWkSDJfIpUUjHHho2U65w1osE>2N+Yqfa~h>2kmTR zdb?sogFi^-PVt2(GQ{3bwioj?Lb{6SkyiE#yjn0G&=u0zpE}6ff{@cTY3q+`f~fX) zZi;CQd~pkyi#5JI)}N45V9AP3sxLMdWScRj)n1vjVuMT4v-Jnu7&{`yw&u_Lzz+7$ zZ5b-sAm$h3>^VJ<_eTuU+riAoMs~PUJ0S}Nj1GxuO-&F#2__boo22xaxPSf#v5j{{ zR>h46Bo^w1c4IVRkJ%j8R{$t!OGDK45X`b86SOHp9Gi1jGW%{KfJd+7UQGToD)H!~ zjFveFW2lti{bzDc4u|Tx|JSb%#FZohs~0%6j<8(7(Q)IHOk>~C84rs#6WGtifP*rr zXH#z%=^|@RVLF`d$(}FRO*yfGi3C)^?OyIyC%dOZ!HNNdf-!O(4xj;q z)0syUb}2b=D#t-b_&+h}vLgvQG3}K(nCF`IQDEj1^AV1hlQR1lp)T!shB8*boBb2S zrKLq>c9~WLj|nDg42EOqF%HW``|}Dr+2lHe8JCgn!DB&PINb@2=R{j7(W$fV6$i9! z*z-94a37Ed0kMCP@-MVMt814{9u z0QWrE*!Y;M-$^+M)R-ew%-5O0*h2DTe%9N;6*(h9BF!85Ur+}@WHDc zX1I=(t|9E4@pxE{o34j;9d!3fIzb_P$xbm^Zy@Z1>roFhQId}YxtVY;g-)8x7QG3~ ziE{O-GaeP~&$#d@Rc3rF$Q3}#%38a8O9n8A#ZmPLLaTrITrcrbs;Xd*MyJs!%Ravp(s2*NjICYXy*-`yB< zKM7eBK`xeQsj?t=AZIJ{JeB`uNh}W!u}KZW{S%XKC5{(Vj8S?=i>;5W^}wu;nwV~+j>h`>nEVX1D?$gt!cI%w)t&`(jd3CGwG<@MHU-^{TJ+s{f=-T`x-a$RicIpy8_>BN?9Jydwodmt_<+uqK2?0smu zeP3@heL&EK;aCaQpC!NbAYj1}yiGEy*#N7+|t@-}<2sR(hQTd3^G*qSS zum~DLJHR(SA?!kzuyWJIm`beqIbcn7R=5=Gj_tnihn(od%&!Pj4c4gIzQxO;J^0`N zezUI~vyGAQN`+ksooj0{sUhR_mU?&05mFsViknVYTYZzioY1R>U8Apt=rjVk8$E|O zW+&$6yf!M@a>(v4juz=QuU=+}Pldq@8GyyeZ}a!;0EWhPh@Ba?U`8A>T{-s8iSgo$`y;Ijnq4MwZ1GJ0N+1gf*-i}wYe*)8(#2Vu7KNGgZ zCr^%i)rD2gNxuRkBV&fgPYSxen1+mx<$r^A1M*aY%r!7hXcg#eD!u7#FK$rJJ*0|h zZQ81(^C`kCP3(&&%CGl`d2%$rltJs9_XjZNq0J_)ieUT-^X7flXCnC7b8TRy$@=gKbc3{xy zD6#aCj%u>tqPJjk;cF2`3UsAeWen#@XlSxymB1{$3 z2~1)&Nnc<1&mb<8o?C2)vH(ilK>M9m`;#*XP6Bpcg+I++Y-Oy$Uv^*(!|?+{a$9UT z2+XzdZ9Cp26I93~!$P`9caSjGkdtHP?AU%)5SZ85DlP455QsfX;=0xI*K!`gP=!W9 zn2TOwq{eV!4FHQ9CS6QRRIwE22z4~tl9^`FH33{WJQ*cnK2VTHydrrY0X7fLQ5jv* zJ&8kp2lbTMhTt8(2I=HSnI5nfA6A88*qB&)2z0Kg4tc2 z;)xN^5?P&3o;ec2MV0Xel_`$`anmIok^0O`Va^060HoP78o*pl>Sj`!EvTrp9{Q=o z(PKbdEAFd}8e5|}a_fx)wt^}OSZ23Oz0VR^5&X^m|~Q_fL!(O&xt!Qt7TZTYL{PWDn)i50iYpO^G(+QXs? zMr+LyF9>jJ59dv(8RH+q%83?kBI9F0MZd2OnLq9hX7=Jy zsByLJx&1Px7Y}tJ7Pks>dX)~D$d%%)`W&sqsE~}*{?q_wj!a5s z^y>+5IU6Wwtk4KeXJ)+tyF~a;870OYC1^P&ciyp&`^E6XVKM2f8qHc@-9-5=baYDBq5PkZH zjNvz@y-Qk)`;W|@F(}7%M?ty*+5>uy`$V97FrGALi^}7fj+G8P8pN%x^Z1x9n)rV2 z?PpQuymYlZFnTP2`H>!YMvON;}R_s0_mn-91}il0w}EHa~PdNiy5NkC2rKiT`ofK#AdK$aq!IR%*ONR{3rO?tWk zr{<)1O&WK+eX-hU8N%v3Z8yfA4&=5$XOPaeVq0P6YN=kn<26}y3v@2kX9Q%Lccx&M zMpr+EMBf?tJNggg4| zB>t=OoeUe-So1={mMD&$j}S#m)=!C36r}Z#a$RkBOQh>0ab$V$pA0l4oxuzNZf^Ny z-0XjlE`3)or<$Fy@+BZHF^ju~I9)VZV*o~%xKfleKtYa_EjA56-A|k14@6XJJ8VSE zb(R0bysRhU8VJ}TN7koa>py6*5pjhGFETNx$MhRxKi&hq&e2(K#j-az+{CBfDQwx` zecH4`Yl;PL1f&$n0DF!ISCLEqSR+BYimdZGI8<&FVqw6rg*`4WpWg~Z;X29fV}+fA z9VuxD`O|d3B2m_w!$dk!y&ggjyzUOC(2YxyrxmW&D0%3UwxoP#M%%GTz~)j;4Vr~I zB~~z+7C#W8valE1^De>`3@-nnc!%Ye+zsaPRd{vMy7!arseuSQ<{U`=}Rp=0xh{YR(#D{Uo$ zB$z+qkVgo+c&v6i5MO4=+XND!2g|MAM+v*eXjf!-+J6R!bMrOwIm zjxhHDF6H-s9Et>yQ5<~`u8JLZY)_a8?c{jep5^*00-T&*RwU0~g%zc+l4kBFYe*^i z@Lf%a6Jqbjz1#-{(DN9|o>$O9)Z5CrE5k$or-3XZn5D#YkG<5po zr@Rg3GH`#8Lx5>DKucyj;BZjM#J=?G z{1GdM&!uQzX7;$>}ocx4z8bT?dn%0#^k@w4VeoxGQJk=Z14)MYk$Yj146p# z5KiOibODrVB4f%u3cCzaeScH@HHqPLpML!1>^av;n=T4<$h@Ggu)NAi@-LkasLJhj zF6qAd#|%``-L;_$0Jumv}oI83Bt}57c*1X-;?V^f!5k^Pan;TOvMnOQ)-b}f=(>UI4}k+ zMP%oOp`f&t#|Uv5F`{UWpKCM*5~Jej;D5IgiO0mHA?ZQM_~x@^@_$^%i^Z4CNhSAz zYl=2Ma0J6?-Jg~N`2Wj63B|t4=XjGiXQT`@07v#*51)GlVwiC}^o0L)nxsKk(S13j zQC(&Uf>$_ikEQ$n{%?HD^u_*yTvbkInq#RIp$o-_UTS{$@yv?~=~f8Cs}?u9hQdjg6vi6?W0Lb;;yPMA)9liR)l zb^PI)G4QYkY2w?>1QspM;M63tdJe*oGWI&jqV$Mi;7CI1Wd{z2aEDFqO*&A^R^p|R zpe}l;6V>D?qo5g{rc8;AMiW#gWIctU{d!OqOIE;oNNsZ==2Szn${F<+0Q0+^M`rax z*m@i0IFn!mM%SCyTY{M{ z%+rX3EEMkC3J~eidDV{F5Uw`L(6MoZXiK)vf;t&>wGDf0TYsYS!v5P6c8!v!wb)L8 zD~)|UMzc{C;#4Pe(hm+6U2qj`RmHZn7wrh@yozJPG&n!B6R?|Y0&BfpcZQ^SSV!Ps zeOCxqn{)SM`cStUkgJ$RXHM$cV!_=3T}Cb)Gq{WrP>fETnz5z3bFcgzL$mf=@N43} zfV3v2o4TpBACya?rQYaqkWN&$jJaYgz{1Ij3><|;DJcc9E_k;92M z>w#Sw6iw;%mB#!rDtBg1+WbEyFr=&(6Voz5n9!uxo7NZ);d0bt;9ic;1<;^c()DX1 zY!0!rOpcqI1vxia7r~y#gz$kB0-r1Fwqk6@b9FPcE5*g8w?q|uqf?RlYAqnnU?StR zjxF{R#3y)K#HeykEkX~MWaNV3z)m8#jief3%44*=QlOKjDQT1VM^{cdZ9|!Pnrjed z;=DY-gBklfH@r=8h2}F+FbUVHvlSE@&@5cvO}Hx8!$!rcqMex@SYm$Iu>#`ek`WY} zueNxv2h5zMmoYZ?_GW-r&oEwmUVv-Qs|oU58%s?Ds?PD8O|$-E70@m;w|p9HG>#6K z1Y9lG&=Ju(8QLPoIz3(%2`5Jmk>9g7MY+%np)#tutS=*oY&{nvj-3MH0?DO_M)WrY zxfX1**kBGh0Dx4JsZ|^+%70_dPoj4n2;efYAsrvP9Rv*@wdv@phH=Djioz}qD_?E- zcR3uyveMNU4~z2ucqQuQ$Y=K>K%5m*k~VQi0;q|o>zPO8KXF{cGVH$s%nLeqd@Iu9 zwSA>(bfA&mKH6zu`4s2mzw3x$WAcS%D z$zVmlgcCaQSm=}t>C$;Sda`2$nPGa|9W)K1 zNS;LlSCKCTa9WMJy3!GUy&S|6v_C^<%lKsS-|q@g%O=I>;3lzm#=XL6uKJ0w$(00M zQXFe>K3|R#1UWlqGp=bLcU1;3TxZlc{2G8_4yTKy6NjPK=7d-Za0_sJv|X3K<6MFo z?|D4{wPJb9yJR=yM06d7ajQGpZv>-Y*qFA>-IU{3nq?#dBfb>oHmv9H z;GJ9ZH+WDXj}@i^m@Qg@Hn2*;-SJSNW*^-}@xHtxBT0!@iH+|DDdvK9nZ!#)+9J>& zcAz_CZhi0`5Vsp!|8jgHIyXGax1QMM0U#>PC4|xO!h;YVitxxV?ja~=B0m|(J7e!z z4?Dgb7{wPN%xU{crqahflE3024r>{Xq~cD2mdwJJEkk^pVdsG5yXJM~((@iCZXt>- zl@6Bb{}F;Ha_utaDC{iLjeDIjE&T+TtEHWliusZts?}VE@w?1~a69Ngea9yuNe*ui zar;va>jYtqrwO}FwW#Fkser23!{8auL6fLf$8pm05SCuNa<^2lV@osg9P6Uv(Y zT3-M_YB&`?Q9%;W4l(+_m?KIL-5URU$^UZYk2kYLST-72lCi^6*Lqf{Ymc2;9x_$^m#u4aW?EPSdPhT{-{p@%^G=TL-5Mc?=vuG$1qfh3q%(m zqDSD0AoaNSbHIYrzQHhsS_hO+BbwPQO&9Sn&$wxklH7vb=_&g_@I%4?1jE1>;k6MTolEiTi%d5X-vc zvAp58oC5wx>m%k~r*9(%cohigG#Sb1rSa z_7!G^VHC|?f2sf%yo0L8qoSP+yNME$2?O^5^8&gB*@d*Q8Ial5O7rEkc!iJtHBNvl z%A<7TSBhH%!N&&F({C#5^2nzGIwh|8*3!U96K*E{{6``o7WkOrU1kwZkP(1n~Yp2(=Lku8bt3ed(RImPI^-MEc*T=a?mSip=*nPb zp}Vu&49j1YdOPC;kySa<=?HVydBT(lSDSi=6L#G(tQO(oGdB0cog@4w3%I)YRD|=# z8C64Py2WzHNI-LdshKvsvo*i;=qiWshi&fH!UNq%~}GnO6=;ryh$gRQVZPN}_<17OXZ(>DYwItJ74 z2O^z0LjsO5MLTor^WoI;8|6=!!(*aI7^kJuYYMwuXgSiFt7l^%bI9JJHEo?DkP6q0 z4L-JW6@>>Zg^Qupe$|P6U;+*^-ha)38xBn z;u?_{w{O@7#3iN=;I(syeF0n~+Y?t}h9G~14wMcayN?5Lmc{tm!*#s;At!k0cx{#- z8ccV1XEx-&txLryq6Fl0Tcr-_ju}FVs_MY~fHJg8$G!y#KTLp2SK@d5%*j-_#F6U%%zi2leKI7wYk**X6?a4qaeiY{Xv_I7;`~O;i zT&86B9=o*WIB2oyIi+~5EywAQ`;%yIuK^rX@)o!N3m%=E4QCH+|Aasn5Sjzr&P=oQ z!{NVZmynzBI=sLmoG_;Fa#MVzpyjrL(Hwd?$=Hrvpma~^oML|wPN8~HIM&`D%(BAR z$->ju948Bgi%qD#uj?jUZK>pjet`-IsiIo#i)kXv5(W|WNjvxC_z011R{UWih%+ws z&!yN^Sk(!9ak0REcQIG zfVm0&aS?7}X*>ELaEd-&-b0r?7{WDUbtw_sQHU$Ui9$n5oO%d=woQ%koq`qzPJlRZ zXp73BIr^kBZxhDA!!m#cGesIv1K5J{$zewlqiAMMyW$nmPPLhHdMs6AS|6~^gABF%~?)fm6>1_>sDcINL>o(L^E;*2<_I^ z6(=a@{L)DiJA%G5fr|XFAB#gp`+Mv&G%v4qHV7r)?XNgiVPuB$>{2Xo4g`aGSDLs6 zpKA&W1jdvh=b1vC-N>UKqWDbf$JkwjbHtYb3XB?!WnxEscs~ETR=irt@eWh=xC@;? zNqeV$|Auf8aUG;PBIAV=ZNba;-Eq-HISM18+Wo5kfWU?Qd*dzz-4wnAtDVu#7Xw;c z(2>hOUpltgTQDV0hbw1^aLF;w!foz7@ycEQZlx{oF7q1cIj94Zh2~#cua7nlkG#p3PITibZih3cm z8q;9>bOmAO+tkZyPK$(bSAtpYi!(HFVb!ZZivn|wR=OIp=tKEouS!h32H3^X-QCvM z{8|8Wlup{_SCdCw59pf7>n=m^0ylwZfa1>ZAcb8^?Qj`IVhuq>7ip0Zzcd892y{)+ zEa8&(W@x7>FZY}ipCqKZ!e+im{ea=+JmE!aN$E1HMpNp}SmYM|FItcFI*yJ6R6BiF zj1%cK3x|C@qMTZz_O9pN3T^(?T!_Suw}HA=Ec(#4^~EDK$kugu8NZ!a$YSsK_^*QM zQ0n`0Pls?GEWD|^X!ylqBmQ&;VdvM-k`A=xVdcQPK%E~al-xlm#lP;(P<>ltr#3>% z+>;@O%e0u@{$3!91dc&DRY2W6Lzr2Mk5V0qJ|@HkYSxQR+uoNGLg$p0Kr;nZqhOYH z^8GoH&IXy+#jipvL`WK4o)s5L;h_%!R~g9Bw)JUmKMd+}X%9oM>WSqZ0fR>rydzG2 z6vC}m>Xf_J*2e%~66PZD$Hxg5?BIHa)FO`y^2Q9)=fbI`7MOp0f zv^|~zvalon#d`@WKb_+x4GCEddJc$UYGD}fh^Tfvo#lQffO1QV+Wox491X|66}Bjg z*H^y)DJheU$nh^h6mrkB(HTp=0)U`%X`Zu)heQ{x(?`n?~-z8{1N=JzO1XF=9m5Vi6{8A818H3`8_xz9CXvNy^ zLzvwxCEDY8(JFA4c(CpVQ1AdQ#s&&k$FCOdrsfY3rLNfbLui+<9WS*q9UA!&kh#Hl z!Vq}c#{gouyqqro352^u@=VQBP(qx9oV>{dWV#?HLGhRu=K2(%7&Z(420Ea-{xe{6 zKz6Bkr>ewqpMz0&tm)$a$U5sN%Zuaf<1PtF;ch_|SQmo(;;spBXXcLVD063acXqMG z3GM-c1OfziC%8lK0Kq-M{fCG5^YqiVFXz1H+&{XW+uxRIudb@TKw+mdDXGv-`m_L; zFEaUk^mzf8L+SkMd;tJ|Y4U@P3x;`5i8*Gjoag8ibxPBqyqtKDax)&G+~G|f)fg++s6AkGm@3A&BUni0FIq4=*LwEzl5Uo2Suu2tL8Cg{Ci0WlZHwP{ zJ^we3jnI;rM}*rd`FFvNPe|rH7j^Ki7FYkaU?_)Sb&TcDhCCdheDtvMpu#S)UMB8A z)O)HRug+6D;_FPwB{J6$iMbvaMaQuuJ1rQuJd(G*T0NY>6sRb zE=$m*K(b6nxI=(bkP;0%+`De3086``FjcBZUj zcM5O>)>>ZXUJ05;)te--#>x;>Yp^5MQ7~64SrqGzcg|J(|N6g+eHF~3k1K!pcH^oA zVNR{!`DesxAm$`%N(-;O$K=(4a`u|jcN+#x^J*`(>{D z$H$a($eecoH0ahJ3uvAqfs~5kH~<&XHiAAhd?46`C|izLbUdK@f$k=7{D*PX?+b>T zY|b2XxrKz88J;?m_Di2oXWiA+JJb?S3xI#gh-MwaW$w{|iOrM>ywtk<;XyKbTr9mF zluN)>j&Evi7UpyjrBI~@)(3FYaWCJcQvP*=5{In{!+zrp0bD(uUY^Nop7&sxg1Xu@n#AUR^w2zybGE6aH)9UV|l zVy--Svl6$pqmAP=f+9h#9a24BYPxD7fLVawD=TBJy%iXo#4$k}yKVWv)6A<4pwiWc zqvxjWiQ(?28J=sc({=!<_tP{FBvq-icPdG-J0srB0@UManDurh?5d(^nw+d1FUTFR zzcMKn-UZS;o+3A7$*yqft_953^Njo%yOp$1SCHU5P7>s$xkt(qI0i_sI3BVmcpker ztALy5?p2T)H&M(sd;Fmwm70v;W0!qEm>iJf|E!=B(1H)w$2Wo0YoCq;bi7*3+YD^Z z`;@j<3xI2kh)bRU;@fJ8LpNhD_V?BjhgmzRq+BP+873DkcnEC+p)`G6aY+Y3v#f@h z?>{F&!r(z%6TjO@IQQ#{94x%m1>l5`Q0tMI#kw86iHoMa6|UDJUAZPD+I4z?-KQt% z)ssp4`4jAhVv~NtPEeMYjNGq<CuB$Q#DRpJ?KIv%i~A3PcKI<|=SdH?)g3bm#{EgPTlWZu915n*x%$|> zhe0?8B;fR)TMh^Cm_d-Dm(*zZS&snBJqQ^zTZf$*2FQ?G;Z1q>EPv;|c|dJDDYPhm#1Ct}G)4;tNqO4nA>oqDs^7;->)T-oo8VCC(J! z=ITV0`BCG4WU#pT9ypc%U4C{?tf+CI5VB{|-u^TP(Sy8R!MsT8@P^hR`=K3A2cMzLE{sD}u~awBV)3(GriJ+rZfFBpdg=&jZUP)}RyVlw^mF z4M-dzh+La>tXll9B?)%I$rD1XazVkG5hW!{jOGi$Tv05mbdKy5lGiar)Lh##B38Hv zF!!onIn?~*;_{vL;56C-mz3}7w+Fm;Au#tLDpHMMe+PgYY|dkeOUs{Zi`jk6a~VK= z>gtUrE{8O0@@qlNPgnZ;X~_EbRoLSt$wN1~s>IjSnN;x46XrNg+VFiP+66);OS>L= zb%}s$wehjxwIxEeqFejv0$hrur+`{u+#}4Lc?w?5L}~Jpl;eTxAY6wY#`NqrK)Rif z=ChaWkG&d@b%$`*aAWzX9hbhli6p^x=?OkB!Y#x&K^rKRy1As8u1D!H#|m;kqB%p zo)QV*{f?}wqZbs;Qw5jDh)kGe`yK$Si?{5bK>I~@B@gn43nSZZGG6r|r&Z%PH0hC& z7BV@#g~<4vCeWq9?GUoF%A)`@4*EmYI9h}gOlF0(Sl}@bYN9K<`Tt4S1;=Y|d$fpl zY#LIg-~SVm2OEBGn6Vj2Yd={)O=1|oV2HZvQ-I7roa-hjNEV&!yf%5-VWb=Jy236h zXE`Krb3X&%{P7Vs5Wf-a{1HOp4KX^N1)(xYCOIG?&rY09Bz$dqu8~5uX&{al?T$}h ze@avy8a?cSbwf7S3nh^(&xz$<1ah{#xWY|Q{7DEkpjXDV|0Z1DWzc>y!=Vi~eYwDu zX6^;h0#$o`AlPM;;fDItkFNl^+Q}z$eEKSYnJWG<%~GyYUIQe{W@IZ1zE0TXmaZP6 z9lV6U>Hl@MoawESK-$VDHRkQ!2CPeV@{U#u?qA*k_qb?6jLB9-7s1a6HS2qLgera% zZ5~K3iizj^cR}jhw!@=^|5HAK1-zXgx?U04gzD&g#RnxN7|cCmto|WLorC508#fE~ zOoZ)G+#}NY;i89>7!Ds7h&v>$NAa~Fmy^e5#h61sDM&I6?~1XXf|%jDrD%`41mwz9 zv9fE8H9iB$vmo8b8GlTyD$2j>i}GCy7MyHvw)?Um$$xq+?)X2a#E8^$ajSm| z=~gJRAIXmn5KhO`O)(lI-<4#fCr-Uq_yNT2fn%l4m@b;`(8Mj|n+m%=D7BN=w*u;X z&mLCe%^v~X^~a2k??pLEkxaz}GYzk2!jZo}?iXz?=yvJVSn(rhvNUIC#H~F)I*u zlG6WY+1gf!xy@k3@I3)JSK_eTovJ7lD@$S?Sh0M;{(%PjKLM@@remo|H&_Ws^ETb% zxRo82q0?f&Bka;L=G(B9ABgo<0n0s+=u5=doXE?iw4+iSu8wzx)h_dN_lr1&{%gtG-b;eo^x~YRL;QFE|Dnk4C zPGQ%hvr4gI$&CSMdzlNxG9vu17IZXc1@NhqKt5MEH@G!)V*gG2aq843i7=OYFrgmW z6w;+aV$KU$qDiU^A#m5N>=#x zj<{n>Cou+_!$)(tf(CofR^T2FYS?}jHD;Dmg}Sv+%*x6KF9~#-%p36KsjUHA6lUjI z8=7@%Zd2mac)kf0Z~?AjtJLeh7VZ34Ls+wuSZKTL%4ZyOcocp9?E&0|d51C-O9-PG zH0{Kw9U#ofDfIZIcZ7s&`iN9S9uI)KWye+^*?CjuW zj>UI|&Uy2K8yeZwxKtQ@w-5Hpkn#Q@({ZUA$u8awJGe+y(eQ~K1`Ui=+b(1 zex}ivMe@}i|Js%R%@u4=d21o{tlM@kDIu}Mn+_?RvDF;(zkSPBEpj6y4U?OMk&kZKr&kEO7z}EohJzVy4Xk6kU*D4 z%9lx|^N=vtkX?S<+zhGe;9KfFg2wPJEOZ5nhK`t zwm40c8WVS{BOgW19%0Z(`V1y zwHVg}TB~bbLda%-o}RpP3pw zD#TscxBNV!Ajr(AFn4{`h{5(luYbhQ{v{dtum(A=fwXc%339U#Hw&WCj(A11YcV8^ zq~rmn{~$+VT&BI-G|_I~v{9Fbqi+u`ALb1`6?t)nKVMQNVPue}9_uhRY;nf%gk3c@OPpKb(Y_}XEInQz)ujn1`WsEtLpdx_ zZ{;L#7ZwXd+MaEP;6tqu1UU$DIM7fzinYp~7Zc7kj}o&T1r} zN8pnc^gQPQx`*&s8Gg-0J3)zghT{42O9b>Fc>MdX(Ah9h*I8O?EvG+M|2U~MWM;jf zB*h_(^Wm!k+|99jO80GBURWUM)aU`j{sx5V7nh&?@mK?D!X>9PL-+d|AA0c|8@G#a zMX`m8O)i3T|6oGU&5<_r6NQ@B+*!-%;p;-+m5zXSTte7oqx&XV&>DXSa?_$D)glwi z_`5KD!eDB@jIeWU#+mH=mzNZ}xVRJV#t3qjlGk#m8+%1T@=KKKTnR!&(v{|&BJxC_ z4K;&m!K;AWAUKtx=KTW9X5DDTtD;GOL(M?!dUc7d_dK{9xwZf`**?Uy>kEL;8V#~f z1US7B@*XkEjnJeg%kp?O6C);|WVhQ*{9}foe9QF$J$SpD!Q3lREk@Z!f}ewKbrS3| z@jiYV1fzj%#l+hQ!accl-s&HOT`wLI$6vtd0xW3dcKsKR4ENsIh{<~tJi>EV0XTM` z68Y5K0Iq^wl!~84J8e`CX!{@U18_q$GeQ|Suig)0PIF9Pzk#Isfda8BUC!pBg}MS_D9N}$eKqDi_rqS#~Cv={{ASG%h0OFqMPGWA-My= zwSPSZNm^358+hDdU31KMf^cr|-WEJoiH4!A{jube1oH@IZ0`FMq#H;#i%IM<+tXlP z#94Z=g!EutNw8~ze+#C$zGndPps5YX`$R7Xj0OeUNUuZWrRd#pac3 zOS#CO`BgyIv3pQPuk3Il#8*>bc2R|$kxCPYi;d3Fmn~jS3 zK7uMOHZDF9?SDIaV)>67U-zf`|BZyWMY$f7I#2xblLATiL}m%IdGsQ*lsjciO7mg}CJ8Aa!pAaIx~ z>{#imk_d|jZqPUa3GxSUTdULk4Mayl_7}5#O&n)b??f^F|G;v4%K-7QZ=fma;I#Nk zK_|&Rk2h=L-fv3+*=iOal;455&m%9wKa9uP8-gh=Zc*YJh0P(}eBnGd%l80ug!B~I zDnFEenJY#Pi>C!RUldPdg!qgQw|ghsnA(q!?t#AB)t={f|7(D_{qU0BDKO6hI84AF z9&^*rV3Z80d#s+rhzH^kh0TfP3hP^SrV;gGU?ixc-Aprs@Fm(TvDPevU50d8=}{Su zo3(s`rk9NNIMUW7D=gqrTI3dkq41+{lH%?wVh+M^BN--NHYbGBWkmORTe{ZVfM#GT zuUgx`+6`^IW#%if<)E_`R|qis`i}854dD;r3=U3?v6~EJTM1cE4sqr>0%8-$MS3<)I}W6yYi`p&`WY z{|!LdxoW>C4)z788;m)%n(j_!Sp>|LWm(k|q00(!d6|FWbCE7D$NfsQFA5EFb&XCnX!^JcRVSI zHm>x>+A9*wLpk}SWZpSdn6pdW_0^T2-BMD*Y~l7n7X*v10_-f9DPjeYWTCfm+TsOK znyF~MRSCjHseVsbov@RZB3n!q?W8NMZOk^bIPihPfa|S?`vIjmX$>$s52BqhF@8;m z%xJD#cpzAS3nN+4Eo(V4Fv{khqUUvkD)7QaLSCG*wjAB-XDsl8k!tR7g zb(g;T{%HR#KX@M~?7t;Z)ta-@SRhK5cDajhI$hs@Y8`mor<2UIVTqL`8!?|C&t_Kg&>im58v)YCnvo@M zLXbKlu5P1Fi)`wD@W-+5W(3{Tj6i-AW1h_mrt_B`B8=Y(Gncpni|ra@H<|&p#urGF zq&mmLLR;`f?#rmb#tx$4M6&UZJw=%lJWedD;Z`9Ocd$9O*^(fA4L|pBzrt=&Eo0nE zZB+p7JvfbQC%{RyV{+Be5o>M@l6%o0-Z8EaU2j9)an!rtJdV6g`L4CX8&teACd6#i zJFz@{M7c(6TR_||^LZ=?6gF!*xyiUoq>Gb0X<*p5T_Xa0{27tCbTW@P zX?tkbP*PK#Hy1#k(4I8D7vT;xV(s{B2PpEbRj0<(9SOS+rg^J>Fg+b{yI^-BN%onG zV!oY#NDzgYj<{lHh_WEin+J8xy4fzEE(DK&aFx&^K;LL%^(PUzW47xg+Y=Y>T9V2B zVYR~DA(TeMa``0%{f)LYQmwvu51@Siq1|;)KE`jkXGu#emMj^@UoTX3MUmr#~3Uib;Tn^vc4oO|K=cc5wVh1$M zq!P_g*q`eeV>CRuMx6xaY;`4&o|cZ;I|1DZaZezn2klBO>;`p>xh_HY_3Iu0($>2Z zH}6N-JVeqyDqa%pBy<4Fw!L2ptq;bYII|xzk7##7N_2xjrOG|je-w6!Ie&8iMVB;c zV9p7JitZQ^%@Aa{VWm4xlp7t54$X9r32`ReW#NNLYxI8x0m+2ZTwFMrunUC|Q9Lfv zZIib5oI`e>0^}?w_44)jsnF)cD80V%Q=oz{n#u|G&=9;&(R{G;3{Z-#%dO`QA?!M}54FIDqYedd4n4@bH$MW>O_4-b+SSZ`B$!#n zE|NDdhGL{33duIBH|`hV#$bynHya-aAxGUZuX04r=_b?6;==2#3IE@!gwfb?OvD+zx^W?__DM8Ftrvl|s=rtDiMW=%~T~>9j zza{+I@(e&1hF-{G`stqwz}}l_;*&E0oO4dm20`17CyP4xG_lxQ@MWT&y^ zIVCB?lr-!W4FLWl9r{^79=JTIz>AHs*ttNosa|Qv1N3<%2{~`5#zyA@xRLt?)dABz z@kF5-zzA)kMFgzD`DI+Guv;9Ty?XHV%LbCxqFNmFSK^oj^fi2nT`M5>KPi2%m|p;5 z)=8|0(1xBeLtwo(CEwj22q`^~Q)W7vtZ@-2g=4r?;tCOY)N@%FKV1xMR&tNY7-v8p za!J7ue`pZa;#FZ(3Q<7(?(YsynKC6VQaHDLQg0uKn*_N&lG*pg)0YBJfIi6rJ{D2$ zUGl1|33u9MWr@HE5T)F?uPOj1SCqFW3&_(Z2B5274M{V~ z$f6d*Mdi^UHPeZrT|_B1@}{W(hRa}kOum+&*~)#F>={110fh2v`HRJGgmC#~@W#ae z<)13h>EpY1LYyd?FPJ4;V)U&L7VB|A!}d#_E3{91Pk%TmoB~Lwigf>dfi9Z-UJS(w zw*zDjw^!57dhUM|NOMtzL0%P9F9F`ld+sdBu))Q7*f#>qpC$~E7r(3gkMIdnjyn^`uv|M?DPS zax-1VJ0fX`bj>mOQ4>iwYM#d+NCnO6cw1qQg7lUvHHy9d3FeHk*h+lg?uimjo)s#* zKNs^p3Ff+S0>pbwI=c&WUDzL`8mnzrdm5N5i4`NBC0tkE?U8;A8zvUje+YMa`L!f8 zqFxi|KH0`;htDg|f#e=7d*P~Rmlk_r7VDunPY`{Y(Q>1Lo)~Z>B=62T+&&}JtZGA? z_Q}5>orxUnauf8!^Cd!0dsiIxLWv+feKo4ik1zsauW~)~1WB6t*p=J1qQ?I=a;>OA9)7U}_bM^bAs5kiy=v@7dt~C3< zCa9h~QgFHxO>p&Fp!5eFY2((n%U6;!spXb`r$E>Xv`fdJ_FjP)wk(MQ@xXt9oOZge z;E73+`&R=c-az|)gF1H|^geOnQ4J-_miR`Lo3|#_4ETtriVq5iZwndr13oR#jG#d! zbiMjRkUT_Xoxk))(B}Twk+JZ{P-c^qr4W(#$GXBaQYhUf3cIzr_>H|9(X&-7&o+KS z6n9(M(^grI#HK%d>QqK?vsY0V{z<_mZWQ6lqyuZ(zkVXnP2uawq3HO`AEAWY9p`-x zk^3?iWE_Lu5K?bo{9DNB#KvEM)mu*wMx}N3v#&v&c2l(j!}+#Y`~QGkts3&Z6Gb}p z=6+L{{gw~#Ysdh z&KBy{p4=M`it=ZzQn#YRtT)rh6ahz8Fz_}}&J5SP-PJU6-Yw82Vx1Zv^UMsL`m?gICgxcD7c`G2o+{ek%TokO0zI+z68^rw zqciqY*fl|B*3})={HH7v&uje26vv}d151|JP044#*#h#w7^Hs47=$qKX{C;*MdaMI zuF!xn(^BOV`6_@-vkK8%DaFu*mxf?9u1$*#mm%n~S+j3P0q{xAcvf4Mu#-kHLDOto z+#oD>P_oxL*%DtY2k53r(lNQRMQdq!P_ubjZ*RQ00whJ?5@B@Qyds1vB~@$E`1MK! zsZPb3uNwQWToA^>V1H9wCWOMVafk(1A?(!Azm45jgLDzl3!(IMcwSQ%>THmAHZh@{ zw>pTYt`$eero+nr+{eqXdteO^mvU6>y%vNSVBcYsbB^I)FmIqaey?C&qd39K!@DF+ zD~|xK&q3;_;@fTlofJCNL-wyj z*g4}LD{<_Ib%ES9r6=7VUkNa~SO;a;{=Jag_jxW7nfg%tdcBfTR_%*ng8U&ev_Z1F ztbvRep3GVP+xQF>jHI^um-YF~U4;%d--b}soR*6vHzHhSgb}go#?USYz8Z120TVtF z<{BjfQ$G8q5sgJxMwHl`Zc4OL%f5lA2p}KyBI6o`UGP>ZH({c%Ifxrn{sLnQNM~Yg zM~Ji}s*E`oO!Y(#%+5d%)77x{}&Y zh%3}7PhyOm(*-$o+;sRM;t4wxOj-?T$Me#TU@)$aeM0L_B^hd=j)6A~AXiM=rj8`Q zM3KEjWLv8%jc95>wgMT53A;FH1pD#G?u2uj$Pq_NydgwA1K*Sv?LpXmhMSyo_AGJr z^a(ZIV=oZ1MLWKj_YWloJ8MnOCkP=0bLc@)dHq!Bm@4tozBgd*Gkr46*hGM1C0SlN zKC}yTQShervM3iDy>HDvZwql(LKYX}_kr}phil{tMCLiua}w46wF%6*_Uf_0cu@fL z=I$`&Y%YJIl*OK9r#1jmPtzD4mf9hmJ&e$3ZfzjaG+;iNxsy1~G}-OzrKihPo`MX9PKBt!}kW$DU;hFtw2T%bgT<&KPCH;Ub*~r#)(bG<2$Cck*RV z4oe#OiNY>qa-=zIsL(76oiS0gE6OY)%>Wp<%QQe|hquSraC(UYOU>n&C{SG zsh}e=zxTyjCquhbylEG^i*(m!*EC^RoGrlddCZ~`N1OufHs{68Cn_g$A&M@AncM;5F2Ow?$2Q4p*t1!b1gALfb)>aZ=508A5GS#KpV^tgt?rE^y0QN z3)zBD_RnV(5|f;GS)@7GH`v!fISx7-#GRWPIxAh1^J0G~5t~r3W^6AbBoAflDM=S| zTq9l|r9u3#ju7{!Npjw-CmUn?Mzki}MRB5hUI@HsqFM`|N7&s3F$xx)2+%dX+#}po zYr;_Ti}NAPVeb%m$lwB}B2~?Q|BZ0HLGZpj*G15-E0c)$zjFhTJ`2TvN=Qz!4|_Rk ze&8iQEj%l&a-yyXIGqMzPzd)$06Se@lUmC43&!td|!nBKd_i_m8 z+{Jd`_$vszvU~Bo~B5TJ9Av&A0~8J)n;%IPK1UYG9}{c#nJ5 zwZwD9*y2gt9iIux{HaW;#1hv*Q?#Tcajpnwk62UpV-GZt-l;=Ovhn2gB?0F4DZJ|C z1UP47$#nyS`7nta9Gu?X2w;|R?HrJpJ1d<@WS zA1bH>o@l7a#3iBTt>u;sy_p{-CS_pP7_%zu0;7+bT)SNUNP$ozPsjFa1myztvJk!~ z+I@rD4Q;^RX~aO6r7!k*lo+{9*{>$)$$P@cmo|;XA9EP-YAp6RVMZ|C+BQ+pJ(u+m z8Ouuoa#wBV(b{H75xy7b;!t@W^m(Fug~W)7cl475;PEOhcoumIKv`5`DFtoWQtLp@ z!2zJk>5IMjnfm7!E2XZ2)CDdQPPt_)cjME9T^=c#v?Q75R(q%Zac0Z#~}9 ztT1I5>WEGSbIfY1B=vEOAh)pYRM~*n;;j62b8{u`QrLZDNJiT&vEXYUZrr|#BpC|} z@O)5}Hhr_*KCUG=b4cFec6%L~`e1-IZfv|M%0)!y%%VAU6I7ub{@;5;# zJ_?2}E9i{)fLApBm@Bk}$@4LoXlVaod z2)mwm<})Q;QYilJ}i}v>_f;*lc0;rbi@z7DFznbJSuV z@ITdOIi^VKeB>{u(Lg;4`%7si^v4MT+#L}9U>d3!Z_Q6iECeSQvz;%%v9g*ov2Onq z(ACiF$24)!XCPD_Ie5INFua`HAIE%7(Cwc5wCP3azYEJPHh{m24@DEbw-Rf8Ns!7Q zw$bH$RitC$?qDDe5KSh?dE$J9-LlPklZ=EP`U=QH5;-XYh|6fnlAiw>9A@G)q849> zAPq^vJO7Wc>n3ll=&#ZfR{RFgd@^v#w*bzmg)@#`RQdEfAd1}2Zu9#_be1FDnEwI5 zLkJb&_~Va|o#n>2+V z^D~gkDP6IlIBKR*_1aK<78wM^vC}LClpcA~#6x}$2t#BY9Dkh+!kM9YsR!Lg&0Y|> zt|m^C5I*RvAkJTZP6#;I#GY}-xd^-W@(dIL0j9{I_@B^R+T^lmD1J4!Q^06!LQEA= zMi>i{+v(4O%{lCk;vtc)Alh@ohsB)p0Aw>m$>I^)W&OmR^-pG4rCYnyDGg*my#uA$ zd5Pmf`8?JfRYBCGjec>j!cLq;p{J&Oj*Io^d^r|psMF>ziRdC&>&x|mNMwM?d!q%) z-*R}YX6hDX4)x)Kg!d^mq(t=rv3`p9i9akGcVDkjc4~bL*d*Fr2#;d zHu*JBy{iqyWg^WmM2YB9eUSg}#*ZTd{`H0U--D6sZgrzdRDV-2nUD6iw$@@$m~-OR zfRTk8p8%o*Op;OfX(Dn-dUY95i+?Q)y^vHRjcuC8=>UT=`nM*UKx2Qv&sqAv0IsY?QC zpmxTi3OYfZAvBEpmnxqigU1)J=FF!B=5?kn*I;xlyL5?*3_K6v*96i&Si;90%Q)P` z9I(`~g!8mF*wP}4m-t>-9%q^e+GDTfK-^wz-PyVgTONSYws*&<6-sQo&W;C#Ftei} zh9cRx6`?Z!q=kH{Xs2Zd3m0a96GYpf6m>2W;EpKGy#D4PIb}b1Rj0~AGpQr0t3lDs z9F#}I#UgTNPg3NXr7~v>${JSFy z2FWH2;lXFZ+*R=Aq74fDCq@;pw`o#VFK83!Qcvz35*z!3cz-cw?bvHHR2JW}yY8=Y z_mu=;3ydirqKP2Q5h;8x@5lvrhaAyL4I zHAFg54%HnD93Gik8%zzyGCWG*)MGG(t%-1b*y6D(cufGz!i0yJa<1P)xskccpB8hA zb~9uhV-g=Oki>f_>z)cT9$K4Y<8=tSBAj7*SY-aRZuy$=KF~YBTU~Lsz#L8PDQfYz z^*|_~?rmf5^~*PvZtajz0FT=?03=6=Sq|8cu#1Sqn?5Sq>9O`}=H6jrklaLh`xqw( zbJH?YGhVClfuQ_hrBjvpy73{Ll$JTqqisekvgUp}j!@W%=rJ(0-YxlW*5z1cbN)7O z`vt$qh2_Z`)6>+V?&eCEPq)HQkoN6~m ztfxf!d!!Bc(2FIv1aUc#cq5I!u>sVk$c|*jRwYJ1Lh_;br~&BFx@w%V4KbXrd>v_6 zUAb+^h?B64^=8`+go5H!A$k?gOy?>#ZG0~i=H6pNGjVLb9RTZtrDqTY7D@Zj`xIX9 znS%%kPZZ!`jNE>a9ieiqIYX>2vc5mTgJTvwJS*5aqSnEtvll(QcLh?+L7MpBod`P@ zH2M*TpC`cGslUpBuy1E*=ZcQsN4r4f{@;W9n$#uM+ZE7nOeDcV{kXvr^!wfT%axOQ zA$kbb*Vsh3o1VG0Zsh%i&^#Sbm=5IpvD)rnE@0wvtg}Y}xEyAUeRfX(H@L4@E|8!=ZFmlB@1l8<9Au48*d{B@(tUHE##6Z2@Fbn!=l)Rf4%&qI{C3>oZ%y{=XjB z67LCjpZ9f6OZ-QWdmWbzv0NKuZX8~X8Hfi3z*nhJ|4(6Om2Rih*-vjT7;inJ!p=5! z?@3@zb27)plSI11F}Hsx3Z^7Yd1i;zJE7g!cobBUj|(BbJdw1=kD_RzF5N1w*5$uY z5MeIAK$Q8*YCIwayP-?hXK(e0fYP}K*?_VhT-?)$g%FfG(Ov+UIXE@u=_6R5z}Ow^!;CNCyPfi(MJyMFWP-rUK1;`Cvf=SB))%nRi%~qp}_gO$DK-x ztJRj6dm5N?YfpC`wK!god$=Ck?&dMw)bXbS*IAJq^y&a}$Nr#hT3JiqaDbCTp$*j! zj;}4T?14be6&Y(64t8sCi7MW_7`NT-Y5rxcKnXdr%>G?qTJeA0%|&0Y->F--hXBwbIB^9;n2 zhe4TH+%BfttsX0`dtk>yJqLMJ7fa|D^qy*xhNkBc+O1x6hO z>^{U9n<9@n8lZIQv2mVg*AYi4(SKY?ramicV}GPzVrdmZ{CtW?r`j^a?mJqK2XLwb zGTPqt1Zet+9=2#xxIV1ekVx!9@~h-5`hbk{-~V6 zcP@ve8EG)i|L2lC9%tCme9{2;66gB=IFlGI5>icx)t?dK5^!4~%bR##kQ0|qpIk-K zHnW{oKIPmYor{%)z!*fsamLw%J)4dhA3yvB%2UbsSp8f_)6QkY`Tk={6;-r1{tDp+ zk)N?70zTmaFw((1tWu53MY(W#i$=%R^DYE(J4>Pxm;MbpGr7_%KPW3+4B};G5Q~{+ z)L?4z-}=KP;O??aCUM+lkZudzIo9Hy%K@AjO9n2%I-~CjAQ~1&k*6t`MmYU;^Q$1t zq2%*YYHCc7SA)V2wncGh7<0GUPmRpJ^^E;lbZv1f@ph4_;+5LpxsxDLog z?qm`E$Muk|J2#G<@t$ZiP|~TYF4KM#=4wgy#VeLK061B_d#jm#H9<~?DVR6mMLQjy z^W=EKmESTqmUOrhXE%3<0H;$)YUh>sNKh^h+l{#W=8}%zl%+hszXi-w6K4^5H>LnX z0-YAyvh?og2@MFH!em;o{jJ0@i+CPnpuHA92!X?TIO2rc9ByOG?0S1ikBinGnPhk0 z0hH&OBz(M30N--Li1q$Kn8r-~WxhMh$JA7M>d`ES|I>2Tl?mi?W8zQ|q*5E|j%O8i z`E>n954hlNkle+Z@p5wyr0cF_h%0cI{j|XPxfoqqS!*XR2z%XIV#`yoyk=D60D-yn zvSaGS?kn+<+N3siaoq!eZVH^oHb?Ix&~!E(ZMx%3Q63VAm6IK7()4*)xEsE?H+hn5 z;|8i=^3s7KIohoa49y&>!Lj7vM~Ua!qb@hZqH&@SYVVnmhcHV7rtemU&5#fm5=P_J zRDuW9c=jznH;_s)bBR5kC623u zm*|1GQFNZq^rVljAfFXRw$jmyUq1&y9?9U22}h(`CB1%3gQwf}4~4tiFgmac(WTFb ze*xx$7t>X;8R8fC7TedD9CQk@stV31WwPZagu1uwUIg^O6)!Y*ar{{pxOoDD|DrJ}tij2#zOzYpaO(Su== zt|8$n`}GgNseC&}p*s|Ip&MGUdT24``jCHI=QqG4`Q~D=&wyQH zwy%A0+vm`vi3<~63w5}Yu5*#XE~4J3lQ3GUP2n%$-iQ{8PVy)a)w*z<5Ixg_tJgtZxasnzD(K75vQK0l8ds!uHs^ zL8GM0n{%SghPo+UOI#w546zND)$FkEAzgCC!B07-mmk2~oXo!%r(P$3a>$rdWlyXiLg((uv4Mj14#;etHcvax z4d{ANK2EnS(Jm_ntb zH}AaV3r;-pMkxShXg}UGA7K}c1vm5WEP*bGTu#OfqMbzt4_SO5vc7Lkol}~ro6Qdj z!{FaQ^oVc@lco6bq-Yq(Eg96C3qZLJ2+io6b1&$>xg;4oHhvIA`FO%19{3euXRYhd z%0iHNdf?JAy%xmL>3hNCoxG;Z{Tm2#K(a7e|2rZ4t*va}maJYh`bKBmEV{mv(xosjVXiE~R$iW71lol}OD^qJbuB+#u-R7k_yOCl z6lz|uCt}xl@1g)=m9{0Z)nZWa0z0?3RAJ2rGK7l-mw?I*%}c=axt1kCT;OhWIHx5k z?&$)}K-q>axfEnQJA5+4ga#l*%JvwsborngDadxqK{^|zb9L=GmM=+g*knV}DZqW% z>V&m;Nsw8>5Ua#GE0k}Tuu$>8Q2_PQ#%u2t37au3JP=%0yI%>E+E%!lY>Pug5x=9e zH=0)_>^9&qK)0m_yd%tIU>~8^ed7J!0nzGo`QG^Tst`I{_r)g)lCT%)%U6SPNjc9C z#oVhyyYXzxZxQfjg1unKssovjFO(M=2Chshh^G%&(*!~ZHcTSSxOS{=P|00%EfC7k z+!q@uNOkZm6;l*;(I?9wh%N59f^rwu?h4aLA?m`s`W!Dc{+5*@e?K$4q`{M9yu5=3 z;Rql%{3NVsZWQT;AE->FJ3cAEc{FL-?2p%kxE5i&EJ!uA5LI;LwwuU2H6*-n|$llSJ2hAc`zh=2$_lFHbbHh_APHiPMKkp{%+7Eyy`D z6y#*)-E}~m@Q~E@w_3M+ge-(DKBiUc15tC<@k;b&0-;`4TrR>{avw)APuc*$43+0D zcDSryFaY^7M`qI;XKc)$W{G_Y#pOcWqPkYY;qXA5x=D$J zG=P;mUK8Tp*3~f>%WMh>lPcY)$Q>lYX(op(+{Mi zx2-=wAuXQRzWj|JYEF&a@tYlhG$GLghhjBRF5zUhXS8;^klYF7R-5w75ahL`*b^mL z-!VJ#hx$LP>)45~tAY@N=haw2u`>`#41^f5&d!8$ch`BgDjH_zjs1qZK<0%?Ze7#G zbvneUp(XmfNVvytuo|=O=I^EThD1|T?gS}Uy8CUy z+>bgrE%wA7y8}2ARKv1^*7)xpB^vK9ae&%)PXKd{;jL5kg+l71j>&%D574fV9z)in zRkb)>pxN8XETHS7=LEqcY4^|aN5b{?;1YqyqlCE3cv+Y8q-tEhHyATjWomq+AhnfT zGS=A#!a2%$RWqg``0EqsVzF`Ri{nJ|MU4iES4GrY+fDZ1zQ2eMKQ3x2G>@2}6(QQ6 zAg1n+&fwC`KrS`Xb-HbQA%qgLaXYGopc$+~T6c`E0=P!H%SdcFP#ArILdnI!tgR+# z(jIiY|0Fhq7DJB9P<}iAiR~^31t2u3T=tgx}e=ZlKvbs2`l$Z-6bVt4@h383&_#f z>0mNQkBg&)=JFyYNe)%jNC}5u7~Ye7EGNW+-X>1)K27oaOy&5mt9P>A%YStY zOpA{db`7|ZMic0(2A~T~-aMOH62{PdC^|)(3kWAVu;f^4T1g(?gfUU1rU^jH3nD?FW^w(*pr+i8T%`U*k?8T^StRfLPfj!j0Kx z5Gzk0>FowO#5Ka4FBh)3?&d^x2oPl-pG<4*;1>KrNu;u6u0!)di6J)F1v z8DQ?@_!C+)Q#cuC-FZ=zGuDL(3;kC@a>i0)GLpYKaCHWQ416 z==A_z*6Ooc64pL1+&t>(^&46j-T>s`jAzy4?29tKBQ(z}pzU0_0f;4dpkMhW!me=( zmtFgc%r#{5K-0f=Gl;X1zLN&b#9RD*4~7O`DeMo>*3+6g&#nGf2HhJg?5ta*@Ef1y zKe?ZarSEV=rid7m!;*scD$IIaZHi|Vtk3LZ=ec>we}KA3T<2i4Rf|Uixfty{JwiJ? zB?J!Z^bw2S3E_t1e1dt5XgJW1e%-8hJ0jd487qq551GpyuCUWgx(Vot2+1>5I{D6g z4?v#EeF3c7i5Ws2t+z?7z3jaJ&K|WBUReMRevN z3U33^)rf#>e^eYTn$&dA=g=-nBa7yXH6MU5r_hG$kap6wf}9+ySaWae_F##FH4OWU zTKrd#%ZtawuJ~LuC2YqikX2-rhar^(va?uA1i2$rV5a^|6#Q%Fy{|FKhtjp5vlgGM(`DJ#ng`NQOyxXRE zsK?vvT?AA0DLkxp*^?y-Q}cix71sgfdBNo80&VgXRL*Y*pXt4bWVR4SjJHmo(T0<2 zQ7}4bkfB5SGi%OVpF; z)jUNdFgGr0p7H~`-m4`Gj-DuzU`)lC?-If8K3F)(QRN*%{4Hw@3nmW{yaq%wRJe2M zif2V-#w3UHqhAMbk8Ed14Sxd?&dSH*8*e&{KZ0m_i?FleajY6f7&i+cW(BLH+Z4{j zzP%;(cpK8yvZr-)tKt}exn~ct?TAYSI7#e6cqS+Q`!0yPg|)QOUUl?)B_3?!%n(!m z1CmFUZtk@m{F^W*kEa6iyH9{wftd+A7dGR9V1=1_?EB>lBvmZ0*EWC=6Go327Poys z4CkZgG*fA|`w+}^Q;%d}Vig(j5n$p)U(EM0VRwflK_q2Oq)P~P`M5tu)xE0$;NXF! z>=^-RgdoECUPNQ1)U{ViX6a8Hi7A_VHW6wp*7t`Mc3Baja^U9w?}eG4NLn~1B(K}c zed@2I^02kSZUYQ$qyw_aX9a;Pl7{Rd#Er|86eo!^UsRU)+y4Di6o z|3QlxtOJ{VOV~wjVqsOs2yJMsy!|~9T<4Y{ zHa&3e9YIbLaW{;6_ZM+`2f0W^AO8p9(4s?3z_GIkrj`M86BYE#$P=P1qMSI~ONtjC z^1tk-q|kMfAZO0TYV5FhtN}FZd7F6RCt|ot90F?i=UC={AY_V)Gs=fY z{tN}TdSTCJ3cKsHN^ey?deTf|>V?Aoirq?A>^n0MwL#3yoAV-^ev9TJe0RS7Oy??Tj2J&|eDux@P3h$}28+Rxhz}cL>TLpDB^Ftkp-_NXh^|H) z{u>ZYF=&Ry_}@CIZmuNPQP^F8nL-B}loO(0I6`E%Oeeo%;etuVpJwCni-9>Cyv|7U z^{fy#subPnFkJn8u{fantQ2CZF=B}ltJ1Zk%P@c*cALt(%2qLd#MYMyCneqSUr-?a8vdrg9xg!D3MY$3}1M203_;cgOGZ(Mms+Gi~Q%E|V0 zD4IpUzZUHT^S<#=VQxxk>LIdar-RqI5x_3R*wMq{VUZ*wCAnxH4dG1Wjd3VW72x_) zcLpsYctI|m7IO7to|X|<7_FIb^D!W9(R6CT^fKNSrnb@b(XL}5T#hMKj-?{q7(9Qf z4TLhAF4U%fB;fJ@U;oub!9xs=?!0VZ)( z>S6Z^G!q#zgR;lK!sWh3R8Cx20Z6F*+bv3_BXA8nUZg96NFb?4&oi+k#@2~z|1{Qi z*%H+3#{Hp8nx>i-W zQ-HaZrp=|cbp%~JvfXTnkpdinNx9v$Q$QC1hCZkWZcVh&V!7=g%dLUm(9!^$a|dur zA;H|^3OfzNz9?qRyAuGTa|(9G3X)t_Lz^g)&&DChctK%*hR+`cOzaFmH8H^(iUUNL zsWKCgeb2(X0GWNYYBIFCcvq0jzU-MQOy>#+hZ>g{$FoRNGtZ12E;j=oA>_Majz1q3CW$N(k4N!*c4e!RzRzYXLGVxOI)0tB|fN zErv~XtSrPummx-594sJjJpH5#*Mle$3wA;@T>LE3EmZ4bfw-g%+6nXEdna7nVp56G zJjnKFl}>0E4*T_VCv%!0SUA9*b;qvqH|ew2t;K5H{+neoR_Y<_>?d)bcwH}~`9i6E zdC`KmLr`~^yzjS z;lyAK!WCuul0VWB1LYI6G6vHVeh)V=&Go6Hy)4X}*7kB*JTVAB^;@t+dRgH-K{Hg^ za6iGMG1KINqO-&)E@`~nB-BNa8Y)ZXzl6BdoDdP*)6YAmly9>ODa&0vGPmseJq}5Vwf-E7cfwc=^gAzgk=^D6g^dU7B*7^@#Fe z4dE7UrUg(}7T1orzws}pP2?hbA6cTK+*u}!9pCn-)3UQVU1^YsY zw~z8CgS=Kb&(Va<73oK=AkuvpMQo0aqA3iT#F$ky;&xKdtNz4)t}>S09PQ#5fZQPl zd$3uIsX`nDZ;_Pt5&`O`%%!7_Bb-Mu3W3ttI7bi*VlUpe8voLS@D$rGLAh}lhG{oa zJs!*zZ0hJ`Z=jb=zc`_!+*O&9jKsD+5r~Y3I7m!Vn6b@t%XDynD3`js17Ag5@v#th z4V(a|18P;Bd=es=IVH!JF4wlFL8L6Dc(A)D z3e(3VFzIx{RJpS=9#$|jR96MD;GY5VK}nC{u*uXJeWYOLhSUPJJxaHWFf&uW|LD$f z;h6V}uu2pth?XkPa^6l6-Rdnt{Qmh|{M}@eB zD=o=)G}S%f+!9S^273A;=YizTAxX40U#ALl;`l&geiH4ZywTI#a+N?grDUtOh;$>g zrfZ6!^8w7+3fJE{Wc^8y^U@SmlY^h9gt_3{)kx*$_zOx(xJ&08ccuWEy@HMO;Dv-; z6f_-pHc%(r&jjX%<<2JQzwGT_YHqSlSQxtTwFOi1RT)}dC(K_L>%(fzW)-JeB&I!g(Ozzq<$BYsT`_>p{)MYF85%&{y6FU{0grEW?wS_oiP!c-??g zw+OP#fr8@Io24Pu{C79=hr1?Q@Yqfy9B0RlzQgyTTtF^0q=$UTEsl;tH!`kR=2l1s zb_=JO+X$K!x@%{X)Dy1@%RP{JMKTpM!EOh17Mz!l6QvHzboRc&v;wN7)wt^ph+KVN zkZV{>`Uj{>gyO5D497GOE!|}BKm%#j+oS#Qr4Ta>FQ6z#O}MipK6w!L&%9K?S@h3B z>GpC{B^OwCiKaX##&pC?cbU+t7f;^};mW2-N-FUm3!^5?3-Q0k|4HB`vzZO=;s0E8 z?va?(_7vh_#__!~*1WgGK&mQ^ye=fiKqoHVZO~I#hvL7DFPITfRErJn;|o`{saLAe z+X?U(N&A%kd^O!GoXR108Hkfakn#P+C`v0lP4CUwFx0F+Ecfr2a2R*O-k}I5xEtc5v(y-;|^i&o!u?*u_#X$ zRBrMR5*}UHJ@+uM%ff1bbL~e8J!1Um;cYYA?iS!uJ%yaIWQ%+#Yv(mH+E+o@qXh^oPc)5evkW438Uj25u}2eRdoH6aGo3J zP})_TCn)!ier_(}`X~G`vm>u}h~Q&A#<|XugmbeX-b|~<8^UrwVFw^N@()h|F$MJW z#V$`1bd@A3huv#D1LA7nJgG0X67Ad(?Xnzo3UF7zm^BU)nG1}ESos_~S&*9`yMRv; zwN0J{rlj(_w7tUqN}l%BwD125$Qi*hGWu8m3~bjzdinDZd2X4;D=m=@=?0WQpVGT(Q4gY@$ef2b5~>k9>={|=t+`hB-)(7n^-?j4=nIjiGl7>E%p*^ z&h$>>5j7|{&1%>mtB7#>HgRt3ZIB#lV#9a&*VREiCfzi}mcraNMIeX(RVaz_AUIx> z8lPeGpKg_6{`ZJLg+{Cut0)KuQ9Ot-3O5!}tUzTxaYh5}=;1zD7eBRF>3!n5qVg5V z`t`C9x0$rAtu6fE2PIlEKJ1SjJ_Iobaq>Mr+C;nAIPP#66#ChtvM^#p;{a*mPRgx<<3C36a63kii6cBx24FQhCF54ccpA_aCxbc@i zjB#IqP<*vyd@I6fp!mRCwEWiqv{`RQY_Fg>#I_lm9?psX2jr$>%c99fvV%>8=7MlP z)fqp213-=2InYh}mN3~>d$6o}Uqoq@aq+QedTFvGjOpJ&<*iAT;|-#-ftdMwKsc5r z?OH4?Qpss9-Bd)q4K0g1L=ah{uMubd;P5!!U0C=>!e%^n?Xi+bXT@-=PUHMniG2mS z9-O+l!02y$#O5sa{K@I{p|~{n{|Gy4IWgB;H@_3)&w2fWb?eTV#?~{z9e=tb8;Db8 z2FycS4x4A21)2hDqp*X*b@q%K86VAB=t&iv+x;looujXp{dw9hqVYPLli>Nict>Hk zcY4YSpNg}WLrsa3t`9R#G z*|S;2eivctGpHW5#Ggf}Kh&BBW7GK|)c6!@dxbNfyD*NMz5uj&*P|T+Nk1)ZTw28wzb5RKN3)ta0h6tT{5J|6v6aGRFKP>MtjIiw zXm06RO#Us9v#btbxhD^3%Lf?M7&&~z_*i>k2y&Bx++P%?wz|J3IAm>T>7-~N@q9i+pxMd}xP{eMm zf1Xs>`5+q7y+ORRa!Hb7^FqMOj{qB6#eCW_l8?H1C` zaZ`aVIdafS>?GQ~i1Td|8>cA^q>n)Z%P$n7DNU!8RmVV>|5|6%En zohZ@`?CD~aaUiZX!p_)zJR~V%pfC`Nt_@MIA$kYMA~c6B^m|~Jl&67OqFJ<)Kn@u5 zuLJ4Yab#u9)Hf%r`-^YbJe($)rqiJ(Yv4R5$jNAGeLcrOmVupQ654+v$Tg(&T%U1O z)V)3!>}Ee5`zcJ}(fw?Xjt%^WB0ZGDQgfcZ5qRF1)UD0X;ucCWx^&%jW5RjXoytb8 zgA0V+Hz^ou0ad2hTI?&V-YwY{vSm&?(b{Iwq7LOOdALLn-3 znhk_HOM9nBLuQ&V_wJ-#%W3U8L8MI6F_XQQDLAy^nAmK~l5h{os@Wg@eL~&kxyoW* ze^h`AEn%s}IyBJCTZ6l;>!zO?jaLgMNAf^V>swK7TA7`*lrXuiyDea@Aj{A{L~6th z;GtvI?I7G$q!1QMigp5scG&4i4*sz~X8Pu7vGn!?-9OQ^kh7m%b^xJano$edM`5Ri z-D5wFU7^i2bf5K5L`&R~idqnAn!v z9WYN|*b?Z>eR2arIXd;Ty9BwsdM0QnuGymiST*y}K6?Vd^T}$mw?#N7+)JU&E1Abl zdjaMSW)Vhz-0%l5=ibVSpl`SQV}ZDHPZxk!3v%vBX(JZc8z9diavX(@)IK1&9fo@F zPffRP5|&$@3)py1bgn9w!4g~jdtV?Jh|`kTx!>k$MoxwH z+=MW78&UqGySyN^`4C|4yliuM20(zPgmT?U557aNKTc1_(@9$5t;WZlI5kHlZW^(u zPn#~c7oHB`(%@FU8e5BYR!kzy-_sirrHS7Ind^Xp>AaoxO{0Z5`F=fnjJe8zAn>P4hnVOR1rg?zGYIEpvr^;wCf%zH9^x3fFF!(I7l3(P?vZ9X6vQk- zMWi#95ba^bJyY!8Amvl$)!$2sHydD3faOM>97MTmJdw78m zJ*Hu>$`L@hSJ!F`uj2*yGaewW=uzC$j|9qnjgH3U4in*+qsxb_Rcxu_@IL|MWsTcelj}k^mtDcMHC_`|p10seYU5`~K;x z&eQdlobprWoC2h25%ts2z80mS*~XfA$f*!6G#8ldNpbl5(~2~j*zZQhRGu&mrvEA@%aRz_IgrfVEM=HcYRCd{@Uxjk2`ju)6X2#G~HnntY}U zAk0J#xY)SdRtH$3SrpzBK)yV$7^_`K*zMDfLh+>{&5ObQmTt_D#|~b8A}MLbR*bjCPvF($Rv<6Zs!yuAkw05a);G4&HrOn>G|i zf%t^C9Yi=44xQ|bs&VWMKsh6hm$AW(^)%RUBtLeS3zJB4DDF})Hw1SGIM#SXh$|$& zvFKmaQ=sp4B~67zZz7U2mU>KudQZ3+h=S20%5ks=Hw4~t;+#6!+}IXR-dcRE-mK<- zQIM-DjWAAS2V}RzoG#Iq4*x{c*7C8M*D8P`M?_d1kqDhWL)Fp+$0_1G%@mCRMU>CB$ zQWfQ_3x(NOQmG>=(ivTXx{LFtvapP`pCFoB2X}tem{ON^7K}H*cLh>tr~VV$!&%C^MNeVA`E+T8A3ZB2a%?<2hnc zkL=~@X_5XY&w0`CJW%f0df|sV5GFW*<{d97x5tH|&8s1afn(_x04S`47O%cYnCi)C zb4#r85>)0~Z+~ngvev%I!0M>qAr*Y3Rbtmyia1GSj+^q=KwK!yDG^6WOKzmVTw^3P zIt;)0I#6wG&R%IMvD+Jw*DPk`hqE9-;oYi|deAKY+}gS|+Q@e+W&(V7_k&^0oq8CO_twkO!;QFJP-Z}*>tV+E5B_LA@Qx>;GnP`;V`ix)k59#uPQ4IT=eQAAT2sr9zVrN1hG8Ap&*QA(=VoFCxv_CY}I^PSHgtX=-QZ+8L7t z=J~@OYLEQ3h=yQR215+Pvjv(nT3Xrj; z^0nXiony$JJ=uXDEdU-2G0}bceet=}hIugH2LKlV4??Jx^~S|N0=Wni$3^@MdSv&>?XR_Kodrfh*kkkP?9(tKN+HV(zU_7zZU6DX={t|zZKuhzf(sXBfze}$~DR(gU&I;hcqRo6KQ*rXK7q`yFA5I?y1g1>dZpAskTy?fY z$TR!m9YM9-2ao!t%_APu=K^-M&@996ZX7f>h;v3{2Gh2m2Y`ObD+DJg2>*~wc0}{M z5N;3Y%AEU$;yc;yj)*Dq0n~EjJ|gc#mz&>daJcD>;UbDS7_o4VLKz+xoVz1?T0M)h z;sOOkLO>m{x1ikN)4NPET-|v=Kv*kT`Ed&u-{^fK*#_Pq$c;K-Z2UQ()F0TwM*kuZ zx#9y-$eC#|04kUjo5qQ(F-CH%;{>>rHN(OM7Y8ys<>Z4JUa$m6(foQh6yuI1!OS`7 zGj%TwNhx@*u_Iy`h}`ff50vBHVE|M}FS7h?+2T)5hopW@9pG(Ww&9OwK&(u{%NLg8 zlRQh{buE4tZH`NHrS}sxdgfgoSS^e4d@Liv-{9V@678bR=+uRofU0p$9pvsRO9Y+} z=+aBWt~Z)jr~@UeO5;iaPK)l^$YWoePNuF5Xig%#lz~xvCMeI5%5rRogKBAQ(nuTf-$sLApz8x}gK1|311%jAC&uh0V{H0!l{Z6!wj= z1(d9pf$kd*ln3O5Sbahfp;0$z6mt_n=3&yKta)%hwH}qXR+G)gOM;vhu1+qW2<7%d z@o)lfDnhH7*(`p!mcw$Gd+OSRb9LxtY3n%CI$&;-^zKbHwh=;;v0iq^KI=l5bv*ru z$!crd^5^1}^GO1WL@<71IRNX81N_81b?Yr_~|b`|0%< z$Q{%H58R9xq^{%Os+$+FTL*i(qf4|iXvP+krSK9g3tXc$XmOhCVlL+qN%gG_uOEv1p1WClj&d z4iK&-FZyup0n;|w3CwBswQ8S_ZQ#y8xiOe(c+p7!r9wS1?pD~R%Osvimex9E;-8g& z$O^3k%n35V7QY%>?#e&zA}wrDJ`!15K4rp#y%-HT?k~VDiM8c;-$sxNjBW+e=$7s{ zZ8tEph3i7OTo}D4i027}!kTm8-g^Qa4by%(Bj~=zR#f`06>|iuAW!`?SaG`}%Jg1~C_JC6a7-+ajha>{$RqIgXEM*UULe zc;+xlepBOh0q_o4TsihCLF8J?VGRz$<9T6p6LgljC7YoUD)%XN2pXi93nJS_Ch7+j zcJda1oYMqA@@dOiEjy;J_OH)jJZr}374W>q($YNTB^UCsOEa*mE6?3=Xq|4DFnVPC zvxWb2<4U+9($NtGw8pBf#owrZ#Lgn!ESl5lTwe*H;L^-Eu#K>nPhB|6tb$ekb)jxw zydmg*nGW`mK-Y0fTh|~@kxwdO;RQA`x}_b^*~v*V%VVdIJll?r;T=UPB+N!;O93u^ zr@W{AAlfC$RzzC)e-R2l<#8e=cS2Bs4z#L{Qkc{mwG71R`5(3`aZHy#;n3S17j`=u zhdrK=Jfp{dA-NqBb5DkF)8c+xD{Jg3$Q86#!= z5wUr-h~LESPEQrSEy%S*65kW+_d~nKktu^225%(H1?j*JNqy`9K~x(_I`eV;e@BIyv&SZ_NhS5NZ!ln&E;6Pj>#sfQKyt;2c1*$lssHSZhs4E zDBfoR-Li-xbh9|e5Qqzmv4_T6TqTGC_v`9wt^JB*=z2O!kj#q$-RVgqc_q;7@w#A_ zv#W%txB@n`KbX@;-5b}xeQ}qd+?mG2e?*a6i*C*4IRL^ep41Jyb`)Kkq_mfjc}k~1 zw-w7TbKuDWiv6P>G3QcMsOw6Lck;UW%m)^TgtAoRZXv~&-Yiw4=imZGeGLCT14846XcB#K9u0lYO2dS?m&rft zHX_|Dh~CPw&1yTUfFv81lqJ1gI+d`ZO8_r&Grn{eY zViAp3`y^~V^<)sUj>Gq0+*YTX&~A?(L}zBBXHN%M_ml$R$pu~dd@96gB!5ujPla}0 zFFeA~O1}warQ<1sM@}Q?{#@;f6;6jTXAwiN+wP2ZK~A#7d(g4-83oOx&>S0{4NY1d zQ=(l#GZ@t~pVwwQ7tE#BbJ{4$93{k!*@+yhZiI_ z2sa;luvm~eC+o)r#phUn$WC>!3qdH*6x4eabam1@aKwK~kOvLc0L+YO=KE174ag;W zEPoMUmw3QF16IEngieA}`BH2ng5t9TtFXt5bQZj7gep^4oGl2JA|~gS=TcF%4M96f z_MGlz>07x3JWm8{!rG%a#;b%2cNtOPP-h)Wn-U*)<#?b0DlHqy)RT4ZqUH3 zG44tTv$Kk8X$mE#vHcqz#IVWUutZ5B*0OSRxA&$PY_(Eq>=}5mO$Y1Dcsv6%-1IS%~ z#v64vB!!#Y-yH)A*X9&zPGM@VM;(*RVammBJP{#zt~FAW-gx&W6$#%EeT ze-q>(>4)z~V~b~sXj*wCs60^+d?`1^rS*T=4fF3{yi$i)KGDv_)=-eELqj4RjwhZ4 zagB9hRF2i2E3%OimEHh2UyxhYUT{^%yX)~7-Moj+%UXi+ge0k%Z)Di9{%{SrB1{uW@Z#jUi&RD&kcHOD4r4Rn$Z5z=w9aS;;UAs zaXMo)7Jdhe65$su{-toP0*BP3Z8PqFV88R*Tla*!+PnNi8?;Ft{MYxK49dT;)BA*- zG4{T)V3_>_5Z9p(V~Q4OdB=hR-9?$Clj8ePf+&pk5hp8b-Z6>O^;Z25fc)hBIL3*{ zm8D0;ZlZJ1*&*W%xi_Z%E{?S3M}0&b4|8tbV;_<5pQa+gdqt5tzMVUxcRGa2>nTi~ z^OjElDO?F@_spOA+ky1{83b3X~JV+Cw$VeOAQdVhR!0pa3$ZA*$F7 zLbwV#SRrfR+0@U$Tx8AzG5m`nO0|+6qOQhYg;A__YWxp{UGAa_MZ=fHN0h{i_nyLB z;>P|m+t>XCxchNvFQbCN%T1CPKcYWiG`6_OEanN(6ghd~8unF@3^vbfCeIMSznI1L z#@ixX8XkaQQkUiM{NMPKNhMs_>lws3z6EeMun8_cTm_jE1Kr)_q`0=tcVJYr6+Njg zh2aV;Qo~%G|HAAvu2I;Z_V?uf&-)&%HVWmK`U?Rbz_?q{iAwq~OZ-rerhBz032^!K zQY8xw^VWHFu+-6l30YrjXKejb@o77E*G@aE zNxu{jbBgYn;y6L>;@ly#EQxm2usGq6-Q6Bjeg$*YIxv{yrLNxqoN*Zq9uByx2w}40 z9qubSH2P1DUieRvCfJ0ughbVH+;t4e$z!X?x~kH4Bpvw>(}v5>u`Fl8!DjvZ$&BI;#w zR0Hmu1B7;G>m{dI&x_2%ZcyTZHwB2tl_|0GoP;So+tX3eBf<=)Jjv7MG(ov>xkQ%# zyh@bk0we>BHKf|uEYcZBznm9x1h`@_aAaIukHFOPprb+AVYj!31;@qxhHA^L}gqG!J1%Vbl|hdk^j z(Cn2_D(n7qAuf|vXZAsOuAU!^=1iv7an1roj0&=&)cL;^<}t#A+h-3;vG9VRp4ez< zj(3RnNN(yF?kUPR}Iy3y#}<$Vg(`z5|y3nk05zyS5V;gzHgLJmrovqc@(1#B)^Zb z#p1*uqei{lu&W3OEu$)Sx zZ6hIGEZBC(XChrr1{&`6en*S9yD!K86jJ^UW6>2LoC6XxWPPl_e-h?$X-|m|gKV1x z(slb$()&SSXV}RzJ!`KBnG&Qh4~xjfYimRkB^Fx=#N}y`oObq=ix@q0XU5w!A*79O zzgT=#!t_w~3W#BM5ara^W)H&Et%f^pBNwEvE9`$chUqeXjS(Q`F{g8sJPr`xy7UdO zvBH5+Ydj&)ou#E4Z^c#&TxGQ)s?H5|>wc=hq6Z<8U3PT<*QST1EdC*gwdNKHRTq|uU&n#D98L7hm~(tVOQI78i1r+!o2N9rF=ZSl*riCK z=IEUO;PP|k;eCg`=o94Zbl7H3-4%}r6F-~UV!eq)8d!iciiQMG6WOA~q_rX3O_+$= z=w!zVF+VsR=#eXptOwQs&5YBVd8G4}pggT0CPCX*d&U{oEl`9U>=Dvk$7(`dzkJt` z{1_c9n4(B0H;(=@gp0z~uQbT%ObFG$%X4(CN7&WuY~qZrz57uc_;WpX5jTl2i~CzB z-X>T(nCj z7nn(Syqhp**;4TuL=H!74qEFO7$l>j_N1Wvao42ogg7H~Mz~9FjXAdj@^Gl49WiyJeY$ecRq^~z7 zi*m=tXoLH-7q$X$xsV~SPl(oS`~lKY9)3Jnlyl`mBiZ>~ylwFbZ;|NUVUg`YoU!hE zq@7ufLv{dk;W(VfIU=1i{hzTi-;Mz066X)JU`S}n&R~{j>K;N~b2N3=ljIA+T1GkDJJa&soJ&o$4v@nURBc5sLSpRLh|V4^~ISYh>F{nxK&}N(u?P@ z&vHCAxuavTefZB+L*b}BR^Asn_mJe_S99BQjeu@#y^Yt=He?l_9h$)1Rm!{!Fc^mj za34m>b(tvV%MLc46X~RpMq!HDj~3!_6~N4FwBo4qLd_tw@8~t+0tH>Tc3o_o*aq!h zlT09x(y|7eE;M&oxW+zJ0E}So(G$ZuAk5WCTPOvj| zISs`7JaZz#g_Q3A3VBd>f#esrKNsT4q8Ak>_CUH&_^#xORVM>D)20&g+HFPCKs+24 zKPc=nCV^#hHOBT9(b0+IrcAVx$0R526$z6@tr5>F=)(2cx?+44#EfpP^rGse&Cr^C zfO!-Tq6)#h9_vm4%Ok`-hha}gR0YF7ROJU^g#i;BpE_fd2p1H6y>zpCsUT<1>o}7* zs7@2&>^b^KJdXZ7+#Z!)P@h!cq6&p zEG%E_QVj-bc2)mX{c~ghEv?n49#JHyS96%V&JmJ(Sz5QsG53)`6qqYN&X4S(MrA-d z%5*pLg8+ZT_~I&{BUU)7NI9v`U>j9O%Cd_^A@S(qv+5ugpu8dU%rQWDmXW(3y*Q8Z z(y{f3Y1!aK-Q$7GMfRGgcl`YX0Jk4I!=_i%eRH2EdfYLtN zPsXhx%u+_M#&35la%zDgsjW!b7fT3p36PMo#H=iUSp^NW*z7dIq==xo6ly|@*_rOht$OwiIs$EK4V9HjKWTa zCygjqN8BpNy>_6j53z{MX}&xg(8cGHgK@OzIRGAabh);e@!TRIrjkCr7B)f<`7lw( z2KB%DFg}$s!a0BEZ`X-yTU|bUc0LH)&szO6UH}PK*n5{^B@rZsGECflp~LI};=zk* zk+`Zp=wgUm%GM5ilU*!;$rq{qB?^*SW2v<(=D!5O?c6YKWE>*WeB$Aj^aL3l?oOdD zpezZHNImyEq=|QPO7WTy^AGd-F5NJEAt2qH} zBsi*fJ@-`DWy0({-p_w^AiuitO8z1@HhH~q)zuL0?Z_OYhhB}{t|`9dMTah`%lOXQQ$et^;!g>3Lcrkp`3a#(?Q5P&OQcLMeQ@ocSyhT<*5Tfj1N>Nb!Y@^3aWrt4qANScDmZ=i#_tWG+}+ zRqiROvHVS7W-@Mjx?+rI$H6LDGTw6Z3v=Olp^lNPW92|SA{Uh~k@5ASF!v*g`6PsW zQ;_SZ#e-UpycvWR>+PNzyD043Boj$3dW^8lPN<++**)wInXWAJ-Rd$AJ?gg<739#{Ga69P4T!Wmq1rT49SMuL0nEYd=ehw zTJMfJhH`0|RJX5Vy!(`$D;c$4lIp z)Ajx>%q8M(Q7_+}csB@rf(KdKVvBnr$U?pvZ@kZcFtD5AIT4;p%H7f-d|rT?lEVmd z9P*xcU!YU3GLh4etK458jW`xUPIIylH*Gfyv$kz-34)<=4c8SHJOJga*yxXEKlC7g zv$C$AmeFqn=EkMN(;cpPsEAhLWQPESa^E1({HABggIG(<`42Ev6943@C`{I@NnFm1 z7L_N=ejTq8P+J$dB+(#b96k4N5f7mNqP5QjxWjOL9VV;=*|=Nh(tC|@=TND|KlGAav@vV8Z|UO5~3c*;cK2J2(xyyHpS)& zx}!-D6ubP6mOA!{qWMgUo1f<}j*#N8e}kL7+-phEK!EFJ=IT^`+EYd95`@LCqUoTD z5$pU1A~(CN$2yE;M#>Z#=YkWSCX)ISPdjqs@LwqB%DWcG3I6m9fH^`hVMpR#xVm5` z)ZfDfo6)+HAom_tye2fzdET^+NhOjFg&Trri%&HcWdgVCb44cXsPyCreWw!V33f-( za~cOfUyu!GCZG2LBt0Oz*5PohdG0x(xfB@N<65vBUkS@I6td{`UW9}#^)5On#I<35 z#=xvU?tcl0W|3#}bzUV5Q`y|MMxO{bgRDT?xVXd+PN3N~AcgnX`85Ezs%x8a{8N*T2Df$ zrQZT^MX(i3r;VKiDM8*2?o&7~J5sE9Tr{O*d)?9(U)Mj@%CFJ?ogf!n%DkD?u=KFE z`NI`1m6@7i@Errq*r}(>;I9R^a9T9kobLM`h?|ButT8m#Jnw_Kz`W?51SBH_d1NDI zLGGQVy%|0Lr495FZ0u7IV<*IwAM%eYg^QUM)6SSzYju16m}0;J?9XD%M}%GFacjor zqFfDBN$H6@3Gl~KuAyna7vd5kC+a{l5NCc2?**}AF9ax?QO8wk{#Lu4>U=wGN zM(%?U$9)0le9`-q#Z5UL7UnUBBj)(#OGuhvhzsy7zak7D^h#5_B7#5I631p=6LyZp zdxEjQK=UQ39U{8wRDh#dzu z=Wo{mB*d)W9|~}3yLc1qfZrf<9rK$ehYE5-ryA@qJ91()OcIOq=$POFDUhe`b;A^xIa zW2hA(vH}|u#7o_y9LFe3`ZU4BI6*{hF0ImbbN1oGlD zaa63ZI25hb(-MseI&KxSLEVyEBFL<$aHdICAg?X~M&Y24YAk7@UwwJRQV>?flOBC< zGHg$Wqa&Agw2JQiZxLaR%hN90m?nhURB^yQ+c3fu&jus&Ws0i+=J*XnR-^u-vDjSmubC<$echrLo7O~0MNYRfOlmb;AWkf zgM6+QZ$SOq0L&c@`bgcoO=hgd~SRZUFPiU#}=Ic zLgGAn7<(w}#3lQWPejvlTv@cnyb~cvms0@kahWJr1I+`3K2HlE9jV!GvL<0?T4gJ? zxkxuJ+H|_v&}n2ys2h+T*{UAJHUC<6}_iq%G3tj4xSH?Gl*bqw(0DqhtfoTDd zx0u&6Aq||^Y-9d$Z>!AgS*XUv zc_N%OrcjJ02AcgE}g_Q z&y2|Q`=o5a(dWc{L0v%P5K?5jRtT9U^T#_xIF;%EdUP?aRHQ;Lk`27(vA2X$wpO%) zk7_JFkwVpA{8xZ`4W@i?RuiNvJYiJ4AOVb;gJZ5TAVo-5(6@`o{av@`7=H+H zAL*05FGJvoN)d_GL}&lonn9dvJKAb@iF8He|Bw3@0Zs%RWM=V7R9lKfI5mqGrwM_d z%&@=I|3ti~;WMGNND>a?2XuTJ09@dzDMqvtrk2U&V_YT5jgG{#D?S%Z%t?eF>>%v= zC8L6Bd@979P2SP8fsBPa!Calr?$+2zq!a8xe<^P7g3e5x7|V7SksDcv$B4|GP%npe z#55r=m}}okY}5l$>nY>MkBD<81DG{>bz2M4AA9}5U_VRuY9jo9b+8kQe(njYKvYQ2 z8g9}@*x4cZm{O`lmyle2mA0ebu}9%M`dWqEEjfE*E}4AE%r)@4G_WDTkVA;m;NGSy zJ`qh7QuFUMwTPbf#??4d2n>U3}iGcgw1AKH_w-E~ybM7)Mz8ZO-ea<6{?Bqiym(>{8T z!)+|Zk13p&Vh)>r7IKSPNA>|x**AIXdWBe;C&f3g0^O^Rz=Juv+bLGBjJ47jJ7=Lmq@ExAQ<4 zY8`YO>iCNv3y}8+6(%7`oZ-lJg54tWNg96_O&^d#P;WFH2LYK7-`HKW z^OW^79sVF8xhHUi$MhiQ({~6ZjRrdIJN0j*(@sTx6yib1v^UT3P%cd?9*1M@6QJ|3 zz$k(84hbO!k837o&UO;C8$nYPPcFs6g7UZ?6YZkR%qn+*w~BO%+&W0?apB1zxy?FC zXp4R%z)`q!)Z04?ol+n;@zi$bOd-ye!ILg+_B##8wNF=aez0cI(~F4gmy-jG(wRW+ z6Ygj*dVzDmT=BH|zFM^7AQzx}vMV_LTre2fgsoon?}VK{0=}erceo&r21z{nc;%Vh z-FblU!s;bWqH4L|TlLlRq3MYn1$j>R0*K6O)ZQr6dl!P3Gsp~C@;AN+z`3G+lk6Km z5M=h!tKyc6i%(E$;k32KCH`MZnW!y1Dv}&g1Zs~@MC68Qi0?&FIF#LzmD4Yxof%Gw zD^2X{+Aalhfp9~`6gqgBzhO&Skt2y)M0>0jX9GMulw+35`6GADnq-4DQ>e4(sdOh> zn6v5-9&k7~m1D6hidfPKZD*mpMi7%s6Wv?E+`llCdQ_x~NN+^0G3iQxJV`Y(rjkC` z@mB#lWBE|h;JHnZTb$|FZ&l27HJH3P8#m72*)pMWe_erbj;wIt?Tt@`(et{v44C73!tN&Y30($V zUq`s>kudxoA;hO&#m5S}&vgtk9UOE6G(DhU;^>KSjtKv$7rW!!{AUYq`@}OM+-4YJ zVKSPOeV-Sqw2`d;^+v+3BeM-VaRl~)oMH=4cf|TP<>=f}E`GDaQ_(E>Rbg7NZF0KhFCsqMR5f0{YVjcNIXXT;ft|zi9w2qhA53#^r)spjL!7&xmyEC$CoA z7uCN(KaH-7Pv^UfNV$M3-vdGc8+9YFwFr-htSHYn+X*z|5vok#l8pnTFwYO;Cq~1) zP+q`_8*u7=w_q1gHkI#-%mw78AHB*|?*nmmop`0Ou%P??0+j7~J07F@qDNq!0sG4X z`26V=;>7Xp6MYXsnm=sQ(drqDeg6UEa%V3j)i_m{8C@Znu2|qv0IJ`^SwB7#k*klG z!8@Ms1Zr+aM$z;bVHcx{tp}a8SBMjwf}{*l{XYRnkcpD##6&njHd);FYvrE#Ux4l? zL_%8thaKG9Jzk*9vNb_oLY+W`Ud2da^L+L1qm~eur_p!%6C~g%D7gT`zl+cE9dg8pNpy3Hd4e_?ZR9LP- zV$uIVm>ozb*)}u?aMxhRfjJX(V>W3M>;#eYHN-=G+a$(HbUT@&uEztEx z52+Gsy#Y;mkvYaT3K#7?A=*Tf4TdIdal)GrZdu9BxiN@O1esSd-9-qu>02OL6(+?c z3cB(wsDD2o($z?xBh!vAg^@SLUa{uegq=5c)QyF)Xs0}YL`_TmP9Ffd-0au7@M^-Fioz%=kHW`e3j32%8T%G`@ogUh ziOo7CZ~PI2ha#H+q#ayH6K7Mw9mmYfV@8gModm!z zG+eu4wr?PErKBV>HTs08H`oK?mfu2>T=JYiMPB*NG`%Cu>DBLx1eht+wH_5hd(k!G zONGs$o)T{`ql?!Ttv>*&0LU`p2NA^9ZheWL9Ok4N!+s&`dLpClkB>#WMR+b9;M zJ$@}P?EksX=R9@uZw1CptS)BIftq2>TCYGMP{+k|{yuglV6(fUdk`tXV7w@Zq9HUM z74L}1MW8dqtTRJqABpCy7v& zwDV&*+{m0h^IYK0jfuD|8A{wMEVH^akc{E~HaC!SZ0_b}3>C;~#5{mx!2Sl2sR%M? zrA~J#tZijS{HS2A9#_4+an!s3wGqdT6agm{{F_i`shhY)=K8k;`Ah8#tMOGm4(|ac zo0`4nBTi-nvQn&N*zoKp&|5(~eZ4q8l=}rL{^+7C0PQh^`HuXZZn>c2r}-yZMAgd2 z2B{S-rmh&fFlc5;@@Y|x$$}hF+Z$exsHUdps=KtZAbXja)OK zY$uQs^|YdBz2ssLW-zBdT-Yu1M*zw$#p#a}CMGQ6YJ71Br%fNld7S7x522e!^!6n{ zJeAkZ8sf$=g4J)6wrD&rk_>PY5tEiI(qO{oqu6;VM`p6Z15}hmI#ZsL>W)(~ZF5%S zb4%BwqdvX6NJr-_=!x;pGSIo5F(0FhXAdh7};QV`F!bZYw=#jdiR5mn}Izz1nI)z zY6(^0j|4bH1`ls5#DOapU$kQgahFI^OSjhdW`b$FA^s=AS+f&M65ekFxhHdN(}L)M zB6h4&B#4!MT(T-8V;(aW#`wI$p-7&gG8m7GaOoKEoB$3R0Z^-5^7$IC2yz@0w@}Uh zMu7P=AR&{E2sf_==32L)3i0mh#W&=`8RU>Pif`)bpBi!0C{U`0sGuu06+uT*3653RU6I3LbNpR&t~z%5 z9KZ#@rYbq`yvk@OmjsUux``f)_3Bs~Z%C)-(YwYF&kc~I|LLv6{|O{tj3i>%Si;Qr zJyYV3;|NkVx?3q$6XD{shfEGq*f{Mo-e30Ne)r&sgv}pqFtqVJUJz`ylr7E^=|XZ@ zuGNr*XH77t%Qb@38?l83&3f`0R z(vo2F64}mRtgdJ-2NrsLX}5N}Fr7)UgYSx8*M;&I%(iS~{tWF=P?bVlH4a`61fF#w zE;wd=!Y*w)JH;h8@L!n9Mu);A+m7e|rZ`fRS;TRk^|Bf-YzX8AVK%8&c@(rWHroi$ z1*F|rCt%58!d#?*q=&vSw0Xk86A}KY0&;{(V`Ds0ryJpR?6V2~yAG1aFt!?nxH#QN z4^^!pK~$@qdzK3oCI|eYQB>BwOX?|b<0G$TJ8Vi6=gCofT>K{5t*I?3+S{6yF5Jus zrPV+xKgVni;Ebync*!gLc3S|sbZiK;!0jT0?#F#&W1LX`yNcgLW{qh=Tz{?oNCUP6 zfYp*}SE3@yCBt*^U^Hz7?O};UrF%vjE6FKagS+jzFn&*@ONP%B)?@j_FJfRk8Q*OK zS<4qWK&-JXfIs2Q=Sna8z83|-#WwCT7Tk_-W;->Z-hHXLow% zdCHzZxq8TGI&hcH(dY`H9%Z`6;l&g-UI}#es1lZX;#I|>UacH5cHfJzhai$2yLEY1 zn2X=bf$gk)AoC<79fYjj@`_*=3h`o_)`^ApEg&9jqny<^vs6H(uBj*)pdZu(l#@sF zuxp)c$FJN0BApwD;w;0%+gTadJmPH79ZS?{M&zitU9?M+jQAMW4-3lEUu$-X{#gag zY-1BRl>sI9N5h(n#I>u933Ymsq%a4F&V#CUGa=Xgw+PSEC3i~nx+PmcT(?PB{TwPX zXD8iCJF{544U~?{Do}}&L^us;7SY-ZP6BaN*jqKX^>)Tz1UXrjlNfA=bj^CCRIZ+V zmoOb*s_}t>Zg7mVIo$jvz}IgQ9tGgdcHWcg1a+y(-FjRgrU;=3 zW35Zv*HuKoFii?!@79q4d`)mWcXT&#$X51%?eU^0Cz*_UDzRn{2z+SK2ECLClo%^< z<7ECZ3pz_&ozQFM?FGuMiCg@%2YF4HlSI>qYs#2b1<6(K=8`s+?gMbYVTmcl`l20y z^@f@b>j!ZC@!KTDam^l$Lg8j_Yb-W}Afsg>Uf>SMVLcza^+1u*APR~MB>^s>+L`se z8jDRWB4N#f9@hi`E;3s%$y3f0;u4~5S&2(TdknKvXu*ygr#E*9q!2R6yN5nA!RVw(e!VEyLJmi1>)UbylTc&7N8vo-aH~>^X9c)9)fMy%$m07KDKriCAd8Nr zgt%I=yJqZ{V?|*qIH|6wFddL(Pq()>))SE5MXV+^Jb(!9<<-I1S(IymV>P5pI!jCw z=#r&3k>i{L0mN5E$yWyvb}4%+T!k^~uY7O;=^|6jvyC7+tt6E(`Va_bxx%>E`%oxn zsmu4aK3ToqBhXp0igMfjpa2r!NUaC9O|XWIiDgC93Q%p5cKZr~^4F|arMOXm^YDH} ztv|!zfNt-`4y*x|5bfsa>rQv0Hwy6tQblF!rTPbDM1wKuh$4X&yJhKe~v7U9XNxl{iya9u7#3D9TL7 z6rW+_jtu1>0p>AcxmjZ)4nK~6 z&1b!>t*hee{su&a@hKfE98Xx&k{%Y=Rs^}TePg4)y(lvpAwpZ+EZV$~+&8VvKMHdR z#Rp_VlyHd?09`_lpw$F8mR?4;&GaOJ<_7038M{6sBxiuVCQ_M7ymum)=B6nz_elg@ z9tIwF3a1naAXLV{A!ZRmeFlway0gq^1P(4rQ`X*P)3c-+3b4 z?8t5P2tllM4v-7hg4I$CiuPpxXlsH|o##Th2#EAhdY7NK8wI)s zllpPu8KcfC0G+i<(I>$9w<3CzK>jvC=A10uIaT2<`+P8J&gLzKUr?lB8O#&`PJ`!W zl84rb7Xs13ZT&GtK`{+`+HVv#kGXx9j^q{>fp}+??s!;TwB6WSu#=RLjs(E>3CiDM z%RqC?b1?`h5AibD#tNIQrKyO8L^`rMJ0o`YOF&#Ji7)7fWn5wSqcPq-{2~&5w&;ps zhf5*cdS%u~)%AC}JTH<&dD=`PTv+i^D8e>{nk2;POpDlA3$hf>#&ONQ=bA3os*a zd#1DNF+!*U4uRrXh0W%Yo|a~HS?C)7tsT#Z9G2I(uN8IyP>^HVneo~JkY^iiRs}dQ zW-zWQcs@dq1{~J&agm7JuVvEuK>icmbmk8cZXT{dyO5%^MD@A?(`g9{mvX#R$8=?m zeiwSE!pv6gn52!($BSGKSTqD)B_|5VbxpTuy|F<(4rV6Yc0Ey#!~$lEMxw3=UJ>D(QCCN0X78H-TyY++ug15c-B&T2jL|nkdVEV} zS$}N%ulmFBu;-3P>m(O(@!Bo?o4X&K&|1Fm6|Yt_7#Mw5rlW z%0kVx9ttw+G)UJSAL4A;Ce{HgZ&;tU6TpDrX{*>@VTz5zUj)h*iE`bvuFFA@?!&Lo zA7CaBH!JMQurKC7AD;>#LuL#gf?DndaP1I`(QDN`@3{xirNq5ujJ($$u^DJYP=Tmh zfZ04K&ChM`DJWBEa zM6DArym+99PH>=5^Q(&PS$jMvfNyxM2D{x%Xwi@FGl5+-E%adV9dbT)z7+v{$IT17 z8xbxod1GW9FV+!eXnB_(dWbl*Ln{THrm#za4>s;^|C9gZedL(%52xHL=Mfu;@WjEc zlB-Xcd#W(kOjk;HyU{-TJfZI1+?poSOlH!KcqAu>ZDwyQ_b8EQ7w$T>*MCT4(RNcY zd1Pbxvp{Ob{&;lk@EC+k3fJl0o&RzQH09Wle~Q+}i~mv7B8PqfHMY}h?JTtI2~aZ# zch_=TaEuV~U2F0qBIuX4vHCFo#ja;$9QP!Dx!JR7Ykd0@kUNiDG{nfKp>vz!&a4tY z3dj*;;6Cerq0L0ipXs%(#f0Ua&88D(EF;8UA>J4r>poM&V1pxPQ5!r9ViQq&1!ZFL znRBZCsSPDsL<-$>p>v)C_AqGDeYZ%uKxwEgR(#%JULlV03cHTIcw18&Z}9>U>}f_@ zIPQKC!e!+l6e-@_C&Xn%H>w;@h&BrljoG;o@Bn#4I^QD5A!HTiezs;kRTK7rx9Nt{@VNcv__U6I=Hr*}_2R6~H`& z`nl*Nhu>MS<7%(K5Mi=ls3_GW>g0g7Q(>?*{`^>k}XzncKMBYnK3Cw>-1;+<`b+y&o) zaDkZZcw&g5SB{m0(y(~ijopiIQ+%T^pYqgCU(EClgn6aMO8y|Sus6;IxT%dc(?9O6 zI2hF1oILa?%=KWeEd!rpg}9lLyBAvHEI}^!#4)k>yQcJnKzl4FiY@EdHDWIX{bf~4 zINfc3LAfz{xwfKTRO8s3KqCUvmKggUQF32q@7*RccO;Zju*He}1(}bX7{vF*bpmRm z0=GfTI_kV@y74*A$grg`pt5oUM?txPBTvW^U~ z?z0t|Z+ekvD>h)LX$tU1lv7GcPu8PIJ$XcYD}Zt%)$NVhK82u?4ZLeyQP@3efM;>> zW~j|y^JkzmX|k-2k3>0rDI~H1?)@Bu0izoqNl(reP4YOk?v5`-xY9bxlwzqb9h;6l zrLCA= z5mCo-UR>iSlfYakH01D~!UYhif_PHGc&2T!SdRFU}Y3DHBJW3=9@DG&p0zPvEdr zlX`qD!Wk!P>X`Xw|I36}iN!>?6ll9}i0g}K`G0opJh&jjS@NO_F7oT~H5X~1KbPYt zt*W;~WOiUIolbRE{aWO~H&mn=(*#k|QLLBO{YE&m7gd*z7(T;VwSm9`ezD|?kp6!X zCjVUN2=M-#Lmsm``<2v{{x~y#xU==(_Mc^-NeG;vCy$H|M0m29FfOLd0_Bo2@=INM zE#{P2!7_u{_eRrf&`waoC^oA%2%+GO+)J%KM-dyjel_+I?OulaJ$k3wZ1y<;&EP&x z0NN(cDJZu{xx_Wah60>zwUtfX-lCn5{MMor#B6roTtz}HSm@|*al<@CJTBLhO&01B z0zHlBG3r(3g(Ph@1j#~Y9no-vD)+_SBB)I(b+0JwQlVN-FF)W904}+l%{%H)Xx_}8DZ|sZXL@?iXw8V1? z=7UdmI+z6Ik1Yu9#F(3V(!GaUyb0L1&7!Xv05(cqvJC z8F}Bew_vA77l~s7qn}~Wu2?(W=~t1pxv8(Ms-vBz`^}aGryP3r`0N~>gx%}O%YnPyu%U%DUHI3J zS(gWO$}5C@mkp)MAb>%~~TQY0OqwUq};2-ao?HtB;gaa9m!Iawt=RoM@S1^k`(6dMz=d|5^UVk`ke48e*+ZL+>Q5@Ng zaFRPDlTrs^jQ=&0+KqA|yhx-+g3i)3S=uy?TLaj|kqQk`r@=^=v*K!6rySlyAsWN1 z)CK>35#c%|-&xT)vH+&E$?xw&0+{3x5z_155JfrlEENxmFVFz#PR5xkIen!|E-I8R z*~xf2Phn?}+^@x)qYFS9fsL`KfLfa&9ZJ2rQ6Pmx9lte}9s}WuHOc{^&b0dpbIx3? zu(L^Gsu&CE+;nst5sl-VPGgDAKTQPX;_1CuWP<k!|gJYk`pg9@(S3Heu4N4kF6wj82hGQx2r#cG0j^PH8zk ztOKQ~Py2;+MYuGbO*l}^|52fd@sSA60u%+)&o&u-m1FjGom`U1#BHKTA!|hcvK}PG z(lh2G)+g)^k+qDYOOTnSMFce!0p=~m)7u?V?0Z9~D=bwi+PfU*ZUE*H%i^VN+0Gk+ zP?K^u#C{^g4^FsmC=3IV0xZki!W%(56ZYQidP{&C(T#+1qY+?pk!#Jkq5c_%A*|0=*o6PZaxGAsig4>p()HnGqWy8h z__6VZNSBJ^if#)wb$mULqYiY7Xex@k&scaf!g;A6G78E=bp*u*y@C7W=EU*NsdFHn z5=BvvoVCUuwt&bpG&UA`kQl3{Egd7hCHt)i7mnBaVwtTVonLw{wkzHf;+8^!)YmE} zqHAwc#F923b=X~qWAOqHcTr3MnJ^Iw&6n z*uc>6*9wFmm~6!o+ZSJP_aP5)RA>`{W_Y?f+^bGEw{ds=>F++0Nlu#JPT8UO2rKP8xQDa^3!xF!~Rd<7_PF+6{_Iv-gcP6sFQry^1|W^#6TvCd{s0FU|m4F#iM&dk~77Knf4-F zYYqG?bE~}X?;-4k;QQFUiB%imCDdkS+Q&&$@Mp1%!g*LAoED|3$V&WD5yc#gheh3; zj@j2SkeODaEW#~XdnZTxqiKR&M|nMrjT#GD?2Z|l3L429-U38ZC9IHRw8E+_M>%$6 z>xeXabbRE#>l-1iGx{ssZsJ6t4CMSeWze>}Xws7k*vN{*dZpxOg|$1!jbt-Hr;=QW zlgc%Ms5`rX!5Gm3K{2!?r^|G=nq)VCJS92eGSsj@}&Jg6zhr$Ktb5x#qtVgIz zCfQ`n-d4~pQj%G05;S=-9k;~iBFt|zyV~WX2KTD%fVuIw3F?k_1ehZgbPL&VJlFw3 zS+k+u%Oa^g-lAi4m%|-&obwbW8g&>bO^t6v)>=2&dN70z>n=dHMfo+>-vs7y#Ziu4 zti^ba9#E>LLzo(UFVT4(OKR>13h*G}bZg7?V}-f5F}XEPb`s($%^k0Zb_r#Sc~}+F zCCuLH=yEIf6`&6D^a-6}1-c*BPt%ncBRI1PRXVsiK?wO{XosEf38LUW2hUQRmjA-t z1FP(>qF^E}`D2BC!Y&rBp5nwQkWLk+x{{U5Hvr=7>9*SVqgUDzy9qV}x^TyqmdJ#e zm#myUSQzT9?}U<@yo7Y?D+#9C~y$VjJzR+(lF z2{WhrTjM=Z?yy$$WfB-FK0=JeO&{yicEU2plY2sK+m5YAWHe|iHWXI~BXuufZ`VIl zrMfZv=Bd8@aug1^mDp{6C>H>shHi!9F+r*{{wUv2*!4umA9q5Z34rHq&FRGdQ~h&o zuq5SQ;lLurq!vY;=ROF?4A)jEZJ}-vmivHj_)3Qqv2Z@&>$FpZxp?VTm^;Yx1?B$C z>c!FH&?1hkv2^lyUXU}SfBBy1iNnAc7d7LIiw-a1Fr<8dd{r3iWUIB~5&YAx$a%?4AC zgLZyQ2D;*m2Lw53nE|IsW7zTjj2QqQtvIGS0b1h|e_MDbIuVM>C4=SzM5Nj@#_#-2i8 zr>4{Rwf;9=hNNQp&-3^@4@jqi%b4?vL>js0V~?@F5TEF1N6tKIT~^4OFDSm2@lVof zSouOQPXmlr%Cw3A^RrtL(|tu#4Z85u{&-VV9;My=O|41ICqBOjlu{$hrUfsC$bGPv z`JgE-76Kh4AVJFXqdu$wfiT|5K`5sr;|DisQlEI|~&3wnZK%(3UZOq zI%Ye0RT0Gw8msYVLC#7dtZPJ?p~;vkZm)lm9vR^N=FO{%Pf!kxm9K%U^$m9H3~`h} z1X45lW1M|$5ua0Sd@Pc`P|l9w*AaFtTHF&$Lim>{CazJ~#b@R!#p|MLV^UKwi{hKY zXgnU<;&nn1t_()r@wrGhx8@zh_3G>MUk~ak%J(|AF9OK28)wBgDC`1p4bsk2GTtq|nf z$c@v{3TrA)7DIRCf6A2NGlk8=Dgv!}Zi6&Kq=8ex!bXT&r3>4j_><_oQc0JI9?E5u zK=LZFC;lXd)k1A|fV<=zQ1SGS@wXt#IG`ttuFQlU3yOzDIH#U|WU&2rI_98s=I#^W zl*(-_X-oIEAeV$in7yX%lnvOAR8#&_h4=$0!Mrchtteq-SG;mRfSQ|Z=X;?3Cx$+KafkpHnGJJ( zdH0M50cjkZYjHq)QIvaI(wEGp=3fbBO<18JW_yUB%h;jkAy@oILF1I4dtK2kVZ+FA z@v=ylkV%L}d8G~@GlsYC2%yNEC1Rb237gM4C`f~I@kiuVh|A*FLZ;Z?TVaFG9Kf!;L4AYQOw1`kkl%~YwTA1}Po&?WLhEp8|o`5{Q z&=k?7^S=bSRwb#5Rb$3~J2j4uv4g^7gZKPu?41d2(7IztgufeQozq+ENYd`F#_d9= z{vzCDN`41$O zZ@^0JJca#zm6vijsQ1PF!kjHznQAmV4V}~D@r~BlUw|XvWN>6WE7~bH1mB|+#?#^&JeBAgsnfgBc5MVKtiEJR`|LEW|WSbF1v6H#w$`8Kig zsFX8l65HqMYFR|?@c1e@_Q2kMQV zg;GCVj-!fWf-A0;I7Ebo5gvkH&0%&5amWYY&Rk-_n?9_QSX4hDvNo8|D2@X@0&u&l z)%6;9Uz}YBd%AnFg!C4H{UM+-%OAh1MFD$6HRolNOwPO!dPR!hIXF#;pbBH6#@4Euv7{gAY|NA1T+Lnw{Ph26`y-<(XrYrV0 zh2@zp+k=*3rXK*^uE};!=4GuHrm}&E;MW#QXgUnR*eoIQNV2e>}z%kL3T-6!@eF^8x`89=$&n zK>J9V*&5&1|HdOOGK)QCSi4p}-sphIoZawbBQ)1T3rabj6+*h6cbIDzpAn$e)w`SY zHui>s{FNazZKmV6MBAmgf!!LVuEFG${Z2uS$?V>* zH)DE8;6H-RNohenEz-5g4rCcn!{z~WF<7S<*>Q&uf2Ha1ZIS*82l33{S`?b+1*H2* zjjmf^7rWHPi2PUnuQH`OrYr2m=t|Z>@BRTG7v3+u#N_#$7IOQzMq$Sm=X9>|BVr{1{v0;~99=2XslbBB4sApQbz}UptrgBwbE^RuB zSzug^&6WXoXBlFPh4Q`tCs--7C6o;9WnnH11`#afE%Aq81=P-+lAx6YIVre7ukR7y znwL5-FR!5B1VT6H=6q99*#9Cg)Q<9$^3&$`(x$-D$$i zK{iq)G=%6RMAXp)a!%6&WuOZ}oe$`DHzhJWeEaQbK|#-de2%Kw|fPLLyO74O(xvBdgd=CY2l2w_KW00Ng$6s*Kp z5&oohF&rNXBSX&KB>GjRH~|p9x*`ANMGhwZvrfwLa@~#iH}e#oL6oB62tlrOQ-347 z!g~d{0^GWG#Zemq0x8Vcl*euts z2xFGp);X{=@=US_=PAvJqL9{y48hfGlMUiPk z!?;*&C&F$Xj@a4YcsGI0nO!;~=dU{hxa~F0IP9G+#NXLaLsffGAlZybPE5+XKv7Yx zyzw^?PLll_%iK!>Jb_4?jG0|$$ftG%H4oB5q@UL5PM%YUd;Y@zUVwEMidd1ucyFA! z8*uKf60^`cay&gb!$iYGmp~Yb=r5-08<&tV$99kgpg2DWcE#jVCWh?|4Xes&r@4tJm!D%q z61MIj$c?S_KV8I37e+peff&0FVHX5b5hiV>*eDBhJ0WP%-f^XUL6lFLGM8#BJ}T)o z3D(Khct)Uel8c07Q2SdwB2M_Ehs$iZY!eallENwSRgo?Cy-tJ#(m92lxB&?e-iCl*50?lS8fZSyAI`HpS_$xR{0JuNKpwAMp{|BtD&4wI|M+Bhyjg5&_f7S}*B@gTd{q6;kU(mmZH znVIg%bWbJ|ba8iCZ1Lb5U~!kl-CY(4`yq?VV$1h?>%D!O=jlJ{yj{2I)G0r8>J-^B zZ$(We<3q`+UONREDb#lphk0-ASc2wi6-Hx=2sj{tKI%0q%k;R zP^5$A9V>ZCw45|E6$CN1wlkS4NR5nc85K7+WAX~G)&=pTAPX(|WI;UY+OG(92{p(d zvkj+!m^q!@jfme1u-G{A$1PhQG$q!ej)adE2@93B_(DN{%E}l`_plm(%g#;G)8gh3>sVF6-*&SHTo?(F4I; zIT$7BY=5*6GR1NVlLrx5%hC)?&WAI>(+)COWnuTO9&qzI9`S3pe#x@lY^w;4!#3 zM`35rd_c?oM1UJ2^Ka^_YKZIp1vqPE7=g!gheKx#6O(Ir={N$!nalANmelc?AdeX7 zz#K&uqvhAlh)l2;a`1?#Ag9I|sHX9WM}m-_{84FRe`ZEAmG)|Vv;9&0-(XX>0KHrL zS#Tx+ZBWnyEpRlDWmIBCzEG1a_P54gj^STw48>9{4i}LL1Jm%qSngN=w-X$Gvj&&h z$wmVCgeU&GqD_Q3FQa*+mJSf&M0j-#1|G}ev-v2*xa0WO^-)A^p5F6*UYMC?_O0mo ztrrBTAtA>&`gjPJfHi--C6fQ7_K7u5fN0!-L2<~mXA8}f^@lTJW5ihwLOoTOGs1d+ zfvgl43Ua-+mAU0U;iR0VQ&eJ8(X^EohOG-uCTwXit}_A11bOdM04)G+;Z->wxKxO9 zK|78^61D(>T!Ib^6!$+ZkC7fyq8!%=%6h02UN4AF`A&>mPbZvBJPqfcgK_H_p!5w| z#3~cw>x{A;?TcsfU&bk`Uz>w1dKO@#3s-pJg*j(sVdfs4ic~GfG-Gkzj~#%{Ergl5 zsq*gobCc#4V>uQ(hyTsW%41^BbD>~eJ}+k|?1F3e{)|YoKz*pCmtu$Wz}#MHNpo=7 z`8iNePnq8*fC{fliTp2|W5xo%160Wi5(OtB1%q@_T_YGF!uL@_Y zfXt!QbOzazE&^8L#l|8o7GXB%Tqu-8=PL}3ebbV1tBXOi4u#*xdh8`YontCJ<(7<~ zU>y95$onf3rdsF+0$u#l}M&{t?3?D|e}VP7!F!}Hn3Hv*BE4o)st*ojpr;8!C3Jv%c^Mw|QtB>#iP#k&d!`WC+@PqDkePNq8^65@Q>+F<;J_=n#MM(tv*TZs!qQ1E(ZSKN|? z%iu@fmnVgfElpip-49}^X^7H1w7no#TGFSbBl}$hl43h1f`>G}85q>#bnUU^19>vN z%=og%*+39qpl*w6vvAA!QSs4(dGfTJPQ9Ws7<2s-(5(vtli0aQ)_5H84w2?49RVNs2Qjg+G2f?{w}FU z7{R&(c@e>5M%7|x<7J1p^#?5r&sfaSOFasebqv|ta;Kw-*oHR#DFo59jGhorDcopN z%;_JAcHN?h)m0IWR|;eJ;^Au=~HplbRjwJnPIZhYg3T>^*TdJm@Gn=twM~NCn0Ak|yP%Qi`VKwjM_*_I*2Dt`; zkt3f2au%{uCeP;q%)?~aknAYhg+zmb4ggar5Kq3m`JipNbO~{y;H*MWw8qsh<{wI# zL04PsB@pL@RU1I3fGq6}J)|AqWl^MTN(!gvU(Qoaqt)Om`6tW`eZ1~_gb)k32mh10 z(uw1M8-=<#Os;nLI{PZGf?0a>1lgXT0C`KGdB8|VBg}}`@&ri^Gt9IB-$-a0C)j*m zD#9fhK+#o;F9oP&P8c6Ay^(*9DTUO0QlsDYX8wr`AG)IaRt})DKzls&Z2&ble67U4 zGJ?sg94EZPKZq9XRJFgoBmO4Z1!;Iys>Fi=U62yKMyROe{srQpyR{fKQT|Ae7Q4mW zAkZJtTBys|?Y$h4_iZ>|?tKs!c3i5Hl?7PhT%>Pn!%UU>KVPT>gndNJ_5p;4g~`mQ zONrz^=I_qfLIkqn95|j(IBStSda9>c-W6m%d4&f3rL43LnFZ*^Rnon`k3~EEcC??f ze+bza*lDBE{4XcWMU!V?M!rXdIAt9jqLko_=Q)8+xx2kskLR18u^^Iv@?rl+44Btx zY0MO1Vb~k*>rH?=bo8i0KgwgsYN0QF_c4H(L(##)lVm0eYp_BAQ`>%8pbJ%KYmF;E zh0I1Mu9L{pY|4M~q*xrfiTs(Ovv~C0wNpjP$D2hapjrZ3kIM6P{>EA#&UB3b9yNI_4b_r3>X^A%&L%s#GT)54gPQEu0 z)My5M+=W^5I{=qKF3C0$Y4(c!B@DOYZ6R=?J5#ab_xa!LdKnWl<^O`19qB)joXQy! z1)H68+fLQAi!hXKBU@ti9|+QwSd_%t3Y$yLI8ztcMUamxSntS-9N#nMCEVQU)RI%B ziefq+7PY(qLbYu5V~(+A?oaELpPN{EY^s3ZNB@+^ODj;uytr7{Z)QCjFEJSY@`JxZyA(D20+%(K zuQJ<)jr2SE{i5zVf-LJcEfteaqZZo<&gv9{gL3TN1mtRkqg(-5Z{W`KVzWb15%sn> zV2=D-+%DoKY?c6O2G`y(-<h3jZ7zUp%#jbvPSKVY8ewLz>8j^Lgt|0Lyj;Db znanj07@}#FEo)tbs}c9p%zSa05OUSye2dOU*n(wlr6-nRmmxsTCp{>K*&40ky9N9a z57x&b3lb)My`&yHECgXTIgYExaZQ?yMIkN{O>X#?i*FS+FBLfeWzoMdh#Q}5H99N( zT>2)vr-fV0NCR84r56EkIi(?BTxOvo4AB-R;|Pf$EHNvV<8l%16wVW8o}Qy$&JHk=2*nDE7sHi-S4kbZ;}+mfbFpytxaG&Ezl$ zGtL=Htv?PHKqj0p#J)k;nb1{`c2tG%FUrzbUpBDl5dk*k?%)#LT>f zD-wpOjAosx?Ni(@+WF%8j(RwKDFC=Cm2pddfe5$BG3*l7S~@3rxHqPYbW>;Ksvz!i ziAnyHCY*WnkVQ)@zf2w*qdn&FLb|GQreI3K8EO2eutmpY=0ovsmIZ4FMt3geTMob# zqN{dl%ohk^Q0JVCwlU}Oc`C_#mo;(h^1J+_qy<6`e3T$3j7=kEE_Vxn6P&xn?^Yme zLCWbp*^X}_JdCZA@q>go0WAS|ZDmD(Y?7#OH8j<7MW9pCZ3(951Dl9uDA|&&L>zKv z2NQo)*fQV%3(Fds)W^c86I`2fr)3ogvMu&wGNICMw<nOZd$0@4DB0d(mbVuF(HME9Aa$f6YeTy#@qn6kDTfF$myC5B5^-UQXQ%xTVa*NgbMGLM2*%*|HsX0(l9YW3HKx-_zE)>#5IX@WxYY?&? z9y*eLoIaPE%l$0+C0(9wH(nFb-VRIUF1@A5VlH9KzI5F zqqKgWPCA<#x&gHF!uZR}r+b7^pS6Be`iE}_;f%7;SPk+2gl3%*LnC_CT)zjQe2njs z?4=^rBUx^Z{zIMw*9p6OF(#x$PYZRs>|peZg*Jj#g>oi`UR{*`O;3ogE7}#)%1S}z zPTg^rK(|#HSwFEcBz$wZ607n}^8b1nh(BwR18mjT*_3}RC|36@e)biRHM?#`xFS1} zH8ull%#qTMY9~b#zfU;%V3<*gwf~rZ$XLWgOncp4Sc6&Cb&Rp)=oiQ=!)EVX18I^|lVG&JjP0b)Mob-P`pzmaoG@iJ48zhPGG;!tzl zEkVq+cGtOwlySn%HAnnxORn0Id>>vA=?3z{SV7R$(|} zV_J`4e}Zr!bqfId>I;OxE5}~R2Kw)!E#d;E1@6*|ZVQ-oM%gaMu>xE|u6(z|8%-K# zS9n_4j{oNyGW)90su0<;tZs=5L}WdXv9D%d97}Ew>JnzpI{Fy&7~!s@LU(c!5qq18~6Vz})W|0MFA+{BbOv?4?v5UfP4MpaWy+x+VLyUbQT)U`2 z=$p(W(}g)Lc0Bm8Y{t>ZpDy>ly<;AShunxGnnc-K)?&S#AS^&R$LAjW&H%0m%vb6i zHGE2em?z!Q{ACwNvr)ysIaT?pyXH}lT%0b_T(N4Y#3iEL`82yVZW=AP8?bAXduNRJ z@`<#UKr@6H61(f3-Sb%3Ot*3GoIbJV9)OgArw96Ch6uBqUUjdnE1<{pN0w2tEY`^Xu|RWdhmc4Q zW8M9JgQk~!bcy+4p$uCr;+UHj-ye!H$obCs3L``g>bv455te&V=KgU@7L#Lvcu`^V zi_QlnORQ1=$;K(vIa-?TE64>+&ZNX>MVLRVaNx!SGPDR{))_UV!jXaH?5#N}4|v>^ z(p0;kP^YLWNjK*g6Xv$XO0zGfigpg%lGCB^uRNl{lmg#EpZeMB`5)NsWpt#T6nfBo=;LLb^wQYe)KO?W1ZjrXA2MA!cpH zdWAS=94C}@<2Gu-;0z^PjOifkJW)r)S0Wuq1~gN51QJ=l{~QyJ*5k3CZ~Bs178* zd1niQ*wNlB?g3$ikfts()=5XDP6#OyNMLb zsb%iimnWgu?qTF$Az5kB3&?)tM?qPy>_?*&zY55z0;71YuHn+44o2m1p*VW`2{)=1 zwHl+A5H}TRlj9Z9u939OPlx!YP6ecKup7EbLC3FYn!axU+AI{9n0Zcp8i3iPbtbLL zN`m~2mZq_aXmiPBA3O3GEzIfQ0)^q{X8~GIF;LXv`RPz*oHut^aK+jOfMly29Ltm| z12i5Q@@~Q%lLHWS^1lipCEb~;#Q7Q3TN}g#W$6R?&#Xwz+sjB9txb$~w3(6+(3_Tb52vaR>p>8g+oBx(pVBU4EtAZqP2e9KiM#14WSlFI^TOvwA;9BgS*Q% z+DitV%i zdN9?yQeku3GHOD6EYfNBRp~?VZ5Fr6lfR1`#h=ax#c%<)0z9aDG#L4`Od1vYE9?UG zXl}&6gAlqF%Jb1N=~xJ}k@g_FH))osBJ|U06lvIeTL^~C>6w}dM7Gkp#!CZVDbzp{l zQV!J0I4_7TPX@5O(7{Cgl>C48ZN(}cw*bsB8^UB5!eB7(X`to?oeV7{mJ#B@7SLA+ z8X?G;<8GGBs{)W7k1G}83K3?PzRPuN)>6Bj4uC16AzpPAZWE~n!)3$w6eiD>bTl{Wd?+`Tu~E1H!tBvFCPkNM z=ggZ-uu>7=@)qbRyxWIS$^F?^y_^_Gr_uShJQdVW%ypt0t$kt)zYx;w4$vdhC5nTE zxdI!i1D1WK359Fq%)L|N(&ZO4X8GXxG2JAK6GT&exGv@$mzyDAlpb2?h)+fNMn}3QVji!gXCO>hM=1^q&Q06=^f*O8h9yRiXvX9JfI_YerKU{H!YkfwVJv z?V>PQpsMSSyYGN-7Rg(DlEs{N0uCRt7zJSf)KO%9K(_f^{FSv3B%nj~L+%DLL-cgc z7wPB2?g1kYEoEbC5!qNSlO@DWZ@oZhCJo-CVJFSIlJ|nS9JOjO{wNZGB3*X>cZqVT zSe45e&)Y)GJkF>v{hp4}ZH4>tXnKT?ZP_1$xJ9yx;@VRJoXvDh9Jk2+m#qU!1Ut1w9))y4ndYhX76BGe zQiibM>x&ZvnyoJGCd6%zL7N3#Z`JEgTRaZrbak+Oph#z&lzr?NguoZqEC%C85zdxF z2>Q}(PXJJY>C>xm?o)&I+WYz}Rlxjp{3;DFuz*kzVuW(lL z+}S`Udzg@{Q>%*qDLU)ajm;Msy`LpgL5uRmGkGF6Vv%170Gwu?dbtUfqvi>N=EMd-? ztLEKpvB4_{(rZ8&vy2z9iD)y+W~7_N6SjBFIL!CBxPARMaeCx)4oj6L zh7&$CmlW~=VcBpth|-D0*kH{02B7O)cis4OT_GwF7A3nYY`L(9<1Xo6o6#|$tuY>c zCd8>?=fRAs9$0)c2l3F^C6`6DnAHTi2#p3q+Q^>;G6o}1wmIdup)3(;vos$)BZvmZ znNn|D_YQ>l!Q_seE$Nl1a+}e#uzb7SeA{nJrrZ*-$WzZ zbUk#uFz1F28;gr4-Uo2)bZG*e^+BGFN+g~AkXb;MXQe!#`Ri&S83XJHlCEjgzroy6 z{QO7SS^OXr$>WkIX8*rD9VrVLB)%7tRUcEe9E0Hd;-fqo_9QZ{pDBbj*P|l#Ra$$+*z%jPIKV7*exTN^95K|_&~?g8c(o%+5C=;yvB&wQJBBu?RYxN2LfQAm&YThtItF_nIgv$ zy2motS3qW&+}YTQraWy7E8bnbT`?~p##Sw^UfAR`D36k7>$PJS*GEPe)aPPzflwU{-9 zz#My59#9nFdPFs!Y@KWIy}(S6D0pJTFVIdm9df8G?N|jkvQw*fns#b4&}sghs)aTXaN?dtk2a1PZwks$@KR5<;uO zZkR1XtTnI6nk+}k(Js;_I_MMO&h`U1&D9^7V7TX}P2~ae0l6?bO4UNV9Ipw4M?6?5FoemUMk@E1t6U$?MD+nTDv!!AR~&5fN59%U67b= z2U1^a`%Hs;ej!j=7j7Z?V%~*Klyru(kqEk$Tu^VWu=(O>tvwDC?PX==51pZeRyF5P zaOXjD$VRi+Ola03nBUVDZ#!XbNt|0ykcR|VU|8ic)DFf!7sg*#D^D6~uu(6Na!CsSBp_rz4dx5!cGk~6lfd=-# zM~)_jn-&*0(hVB8?i!n;^gBlGv|UeVR@;27dA>&xDLLK)@vf8+ESL-NxQMJ7XXikt zjstR0yQQ74OL70OK*y6dQ1(u9j|Xxy>_c_VN@uDNjYsIw_W=Gz;a}UIdkBAEx_k0A;qj{SsxoA&64->#2ba2|HI7db*l(rx5<- zo@mVTdxwkFt{ASc`IT4nnEemXWMfaF!$p}__>!}LQXC}ABH>Om%c;|v0G7LSk8_#; z2RZEe|1N;Rsn{0xD`?L2`XCa0O$ZXT^WS(wWFyO}H$NWUSp&ih1p8^T#NXXveXS4~6N0$cJA*$uFaSyy3K2O7??BjWD@oPQk?z&2J7$881V z%-AQlCl7`^g$Rh;nNnqKbJNz4ma&>x+%Gz7rab&1dzaYvPk9_>ol3kdI_n*Hc<76f z+X6Tv1_>$6Nckv%Zroi37AGA{DV2CcC}qX-04aaJ9h7t9qzbQ%G^1JDgJnYR!55f=#7YH(A=!=6D_Lr#3SiD#8(gkEm zREz26s7@7M+7;ANBnP&np9vt95)XlWt1z|1^$M7Z8M{G}n+EN;NrcCu7M518iFApu z6V-lm@!dgOr&^H4A4RLt>%8@Rd9&|o5{ZbshCzyFg#gZ1ar#2xr{Atm@>RwKU zy*4j3itr~j%m<}KJwN{+mG41&=h+n4Or?X0tA)V_T4}c9B3%4Zzn%e`vJZgE!7URy zP@KIlNVX%vZKSQ?wZ!5EURJ=^cuM%^nxx7>JK=1)%y-825I>C!(Bfnr`~yCm~r!Dsx+TlM*zA zMXkmuM;XGYO~t81IbLl8aHC^o!R}WY#jV=`&1Jn4V`I9@?t+{TuYq@>fZM1e|1$Hk zO8eMSpfhIzF2^OJGpVrfMYZ!>CkX5$E#+pD^S?Q!la6~AA(W2YJr~n{5@klv&Gt&| zHlhp2ZJ5;~W+_Q|^!sj5%R1>(rTksK2Mj6JQ9!PwuoJ>LS?_cPt{VjT6BY!O&JykK zCSkCTrDamXty_V13uo6Idx>;u2Ff)Zx_l$V;$zLgnuRr0%vW_P45KJ=$HZEqNd+yA z-uT;IBob(qS@;Pe%$SV)xp)>U_X0Uh**&t{pfGy~OntH*{Zj~g`IkJaCo^WWpMAiY ztmQ$Z7B32+c}^S|v)2idX&V=ZVsR0!CC)~BV>Qtvt_#DPD-5&kR5A;^<3%=_eljmp z4`MlcP4*2=&0}akk({%xJ29rLD1>&Vr8@fx?xzT_wB-hC5Y2IXCoEgWatnoSqrc*? zsd-d(Pnbnf>njg{Y0SgdWLK;;4GJFRcAMMz>xp!uuBAoLT>@M_7It{|6pfIR1||)x zF*pchmTA4xLVPa7b(*Nw z`yL97{4qLY`+LY=pj>fGvy|Jz>P3Rdwu{r5M-?`o((uFQ+d{GyfE)a@kXZR}e^F#r zI8I?o-=jrOIVOp80*pV>r2IjM<$;fd_DVY8*|!<3q8oa(xL63~z|R<0Z!$uTsN;kq z_{Y_k^b6d-C)qiUbo>%V5ivxB%9-31%n;>FG8(&F0hWSR zWK7Cb_-e-jIwx7u$bnZ}Doi6OtA=`9B?_q^%dYt3IEbv{VJY5N9^7(#o{C&Jv5l_9 zQ^L$IeOV)ayxWA@s!|hnMBfR-q6kv-|D>?LV9c(tYKgf{1ajBHGCh_&2@=j2AKQDP zcuM|f1&z;Yr$Sl+?7XfLoF2!0X-how zY@kNZWXU(=uh6dgw2ssP#RI}fuY&`pw-k1KPTdPJa1ONV-kw^0dHo^5PR6h5(bQ(0 z3&!*@eq;RW^=hoWA`FB#RY82z(O`zoGM1*UAN*peTBxIPYAz+4RdIU}iIBVSKGXKl>3`#w5v}nty z*w?Q!!|R2($Vn$mb1cQ&mp4D_XC1M@3~2M!Tj=fQE~5Z5qP3Qm%44qpa&bw*4rWdk z1~WYP3TvBP3FWMKSwIKDj3!lqS!=;MGF^x{S{U_KZ7;-e3cGzz;t8!MMbeP=SwH@* zptanrG-(9ws+dyjcon!))N7ULF`So#x!gJ#pd0=w#I>4fLN%w(b2TVA7yG;79fcbM zsC-%%t^sguVR6U9cGp5@em6gP?OUlQi%13KuPcmuSVL6eV2H_HI8nrWW53ZO!2oCLh9uts8)ea9P{N$EPzfuf06 z=<1Jk|KW)A)crJpOnEe0F6KC*teiO35VO>u#Hh~vC@-jbmWQJgQ#BGe@gSsq+0 zNL7Lg;d+IUh;-O-s|e?zQ$xne?}a!;rq8m>=Hl>s!7QVeQIq0!ky(M+Z>BeXUdf{1 za!%Fp^L<2d^Foi-N;>RGA+=n17xW zCfbD_^G`6!B-6|H6m~J>SDEp9#fL!L-FY-DoviB=X=TCY3-90P@E(Sym$XieBNepR z(w2f5{WL-5V=~(P?zrL+0JSFhn7&1XrGjoLUK8o0b*(SPJqq2Z8r>#DEEfyo6PBGg zw2Ci8@=4O4@%MI*`ETY33^w9Gk#IzV#;z?LUA!a6iL$ZeD3RfQmN54t<_#7ta*T2I z6l!Z6YCdIfXAj~}Hah3R^=uDv8jb+I*LkO%F%W4+! zsz{4Q*Vs`3JPBa&vFBxN&vhL^X3Q6nG&wEzR1UJOWc*N5*hCS zh^$gr7HF-!@H0SJ&br+diwQ8VZJp^vc4;Bb*t3zuwAizGG_6x`ka~y^3zIRrj^n6u zto|GroQ%V)epiKE!KjR4^z)D~CNtsa6G6k7G%^->fuQq8?E6 zPms%y+GgA@z*3OEJx*xkF zNCK-yLY!M)Xq5ny< zi-B?YEW(XOpBD7(@ue_xfuCrp$@l*-N0PM-?wp1AyV@W}_Bhvw;r|9B-Ih@;algWu z7?=Y3STU^m5fD6-3LV`sUPRU>z0qeb`jbF6kuHo(?-1!m)yZ`RY#ju+a(TlG8j``p5a-I#G@@MK^emm&$saBgjSL5*OoO9Qp}JCIGpG)O7XTr+F;KeqFa);XgT2 zkRy>&Ocdf8WF0o0COwtOE{+i_*|H_RUt!uF9yidqKau^z;Yj@W8UMJTcxH{RFCbme z$!I*7OsKz$}$QPTjnCEN43`RI)U0FeY+gHxc zj879rRGw0d3%@075tYhq1yrK(h#)h=<5A2Ko!`Ciljz*;jg90NHqZ-2BHz&Ex?jxch4{05MrTp;UX-q{S_L4aep{IRoGdy zO2GpwyUq58)aba`h4ruq;5|hSxnb=e2mhiIE2DWfDDWcD~I9 zD7q}xX$a>ofzCWFr0Ck?<^VFUxFjsbKBAo`w$$jK7z@9eGskKL&IwRz#X@s|S@3cn zI3iAJ0(#D?9<$F)47aajSXD`WCgz?ePm2Oc`8yc%G_l$sI&W!lwiagoQE=9H%}fnI zpflw{K2yUJ0y1f`CDb|GO7rF!aJqtxJl{Pl%u{j0o*5ajRuoQm@9RX@^^J&364DGT zJs)&q>B<~4-OBCD1!r?q?z}HX&JXIATE{YUdXrS=dsn0Ip z&V-YESu0~sFH8w(U-m*Xf{uUE9r$4j5h1HZSWSvkJO&CCZ#1e?6n0Jn8XsG-oV65Nu=jgeh0Ex#zZt9URS_#7#~$@g>drJvo_<$|)w zq@a_Xcu|POpH^bcZ!#mp*myBwz%g&g#DNOCrdfIP#PgyVrN)koRfiFDZl$aq{v1H& zNam6Qv8Mpbo4ab9t?G1qsU-m2T2du-GJtT-u#;fdJkLaYF3PE60Gw1nt1p>D=`I0D z{UFH2Lm!nKsw@TK9FtzX4LD8}m<^7cj@;KIyRrTFT%@^X>&754*Khzhu?xzfqRg)C zh3OBdVRjMh+Lnzg7T7w+oO@|dvyH7lk8HqK66EBuOc*^PUJ_t#`g(h!V;M*vZOQD> zHLP1`8!gKpxtx6HMPt83u(QyhAZPLiF9+l-P>f-WeqIyMMc!ImC%|c8szt}aoS!-R zy5)i0&N+KxvqSn5eg|guYW-8=6OpurI{W|gRv_FMX|#yOTUp#9&~=~--K4tliXi3? zB~f}(@-ab7*BJtWZl-?jIVfo|V=DrdVD@^Xz4w9I6-0?FQpcK?szir+8IiTAJ0FZ>gp)@(OWlnUKmwyjjAOgs-u#>Sn;TJTN9xOzZEajtcXt&NjjLbvqK^~ysR}i=AvXh{vevstec086?C4oPTt%(dJKT` z?5&oy(K}-tNY+|764Em3YC(?FS{*1>2cu&=2rTHmKL)5*MOsGM{KPxIrTcp04-@#` z0$^>W`|IkKTMLDmjqvO#!sTJq88aeo z7XT;hmg<@(?-T8u(TK_XRkn7!Oajfu2{*x1^aMM99i{21xQ`XnCBxNyTZ~-~+Wbt{ zHV<1nA&rkJIDak09qR)j9n?j-F#C>3H+YUxX|B{DEi_o0p^3*yF2FF5ZbvW2O?D5u7aFvOUvkZK_u-|u2<&%Jz-?l-^W9hS7ihm zkM?*{gn7dNh&>k;6$|_UtkEXY^u6syAkK|xDk(dj-57}EwdL7<6T;?}^=tpMI7Wb3 z>8wwU+eD`RuhsYPO`+Ufc|RG|iB_kd3v~HfCbENEVl!y-kZc^dBPqn3`k6jzcX?qj z(ThK&y28$LvaZ866TnAC6;0-IS!B!{V)j208G-2BEY=Z`4M@o)ELAvsb4S3&u@WOi zWZj&N38%W8BUQC$eL?@YEOrPim-gRo2?$pc7$JU8I4dkqarnC6@T~yJor9#lcvOUA za7387!(L@;KueURGFBHC2%vPyGPo2oMY>t%Mkz>k8&J}fVN(=DIEgy9q8Ky_w{=VFvu!d$F z0pJkg&roobOpa%YP?OU@QmQ>*7 zxK)5tf|o-4D4JY2IgQzN$|K{;hV$DH06%Ec%CW2nSe>5EAZ`@Z;0U9PU3Z4Ii1AQ3 zol`elt;gO%oxNINEd~TwgdK(HIw5^ch~>=5CpJdt!UU0Ro0KzOW`tf^izj#CpR83U zLpF+d!>(X(($6MGR}XF!Z8`LCbKBPxZrTmdjBpE&J<}Sy=PAi&G{a+GyeG_MO7FmZ zA=<2YAm`9~4R!NJtg|4L-h+99fhOT* zmCFoBOe1yzz4YVUx8_R@nz+*+W`|ael(}`Jge55*Z~Q32|1(J9O%cniy>kc!Xljgm z39=-xJK@$%rvMj;gBK}RF5f3dv|^#|pV>D@Si)n19bX8cHE_B`y==H2R5n8-r;&08 z*caCcrg6258y`<746BS`^;mR&2s6P%gz?Ht0{FL3#0=nd5w2y<$Jqujg8$gWbVCKr zK`AyX5YL^o_jCzXJ*X_v>wSw84w z^+ur-lzkFw=%+>bGxSc$ir^bT=C+Od!#Zz`p=B`l_;yV99u;Xu(T(zQK|CYGAC)nh z<}kPv)7x?kT{ePcusytqaXr&D_!~@aJ#c;vmx>3MT%u)?jSeWJ$>Oy=R>}z74v$G9 zoG&g-b>>)$orGl#Ku!;7HbN{3T3X{$d5=(+9rv9K34JlG`Kf)m$EiY8<>SZ43(bE^ zg|t$fuhaj^?)+AToe8WoTx)(L)ROD&FUDq*A>Bi0iv8{F>E?w%_fs7272~jGBw6D7 z+2y&rh~!dXZ-{T0*iJ~23N>_n^KUtYX9jpih;!x+SZvx2>73I&qGU%fTMuBP)o_f% z{5+>1OEh_Bq$$M~mHbBHT=|n{Jci$@r-GEi3cRc(yl>!@{DH28YF`fSD!C z>2>u#)^0*w+(hc(aQW>fgqd@Cu<1QPmUdS^woCW`?#q)$jVRmmj#yWi+1HD4(IeU- z(Vnmpr#2(-3|mLMCfdoS1p}^4Yk9J0(mIoi<5lZmw7GT^s{8j7b{4W7jG3aH0zE|o z_M<{98<~Kycj%5~raFZRdaQ8@TL#@#c31SSKM86~ZgRm;m+@D-P$$H3xt>yJkM8Ej z3{%W1T%Ho-hLnyllgZm619^)2u3xT>TjNln&RS=;s_`cUnSZnZa(p&y8VEvT$smu4 z+>aHIb(u2i6_Ng;(1t^WO57yG$rieGoKTHrrUS8fF80Lf3O3plo@9RzjWmmck-6V*DvXj$^)x9Fk3YUVr<5AIMjdmw~RoHcteA9L^ z8$I)vJU;VdUwk9l6~j!;PDAU!B@P3$Fb8;TceqHi)rL7QehLLxBIt_MIEEb#l4XS_ zHSGc8Az^MXs&g8uH1yjZ0cwVOLE})-4BBXyi~Vu$kx**I9p%{eD8g{T#d%G1HIa>F zg>2DL_W_?SxKT*Xs&wKtQy8gKnAvYu*t{|V$O>o0qd`1gpgr$HJz}Noc;U($tHVnb z_E)Hqr4nB67!V6euF^EsPLF#9@<}V^bt@iAIIBGN+~qh*fJ=&9#X!t)Tu#$y*_Dbm z=W;KWv}B7M4`#`?F#2sNG82}jv5T62e#~U;kqh?XJ?n zG>S7doS6bEySG!PkmT4l;Rsf7A?AZyoI4f{XrNFO`FF;YG+qx| z<6+Urn~PnQ`13gs$hRBq!A%M~b-b{#|F}nhQ{oo!*tq)KobE#>ADf?-f1rgAt^a%g zl4jxG7sD>dW0$387%AG#M6(elgxIgej?Gvs$)`tAbXLT)wjJ^}0E>Z|O-koQ1d$D% zrwcJ%gg@Zufj!*`0$fcT70Q9)MHhm&96fAs8tSb)SZX;R5#i>G_ zozBF{@#o6{D6mw+@nS}>=$R0Y&fp(2La*0q?+GEqC6CUCDziH#KULIc_t3H8u~?{^xx zTt=O&s3g}ia&*i&ldvVh^W!l>q?%^|73l9pxGkgFESWZ|J-S3so9j>%B{i`n4#>j1 zoE@hs?0m7&<4&vGf?VDNH4C?PuqTOsHbIOmQGX(%cv+w;g!Pv+t?vslFD&*@a9y9% zs0;AYCmP0kJ7{KSi^!Ue_Cj_TyUh)t@QPhbT&=J(KzYJr@EC}KxSp_@EOe@URd38= zp)YLJ4FTkWPs7+0@QMJ)8QHsq}_ok z|FC-;#%v~@SD4|@)86QNq0AEc2?pma1i0*6Oe)2mqOC&b+F z2vmnAr4zDm810yTqVOm&+0Tz76WRq zmJxB4fNUv-PNc*;w8>=X6XC97&Lr_}t}81lNcw|Gs>}{8P83abPRUmM?00i4xlh!I{r$oup~>2@JeyHm>fp#=(!cnRn$V^>Hp>Xj*0GNW z4NK!E#<>c+u;`!IW5;4C==e^gdk}rT8W+xjwrpiQZ~viB3v?nZD^NtdCcs}| zE{crONh+MnyQ}n_ogw5KZE=#>H+A=Bk^>a_A6zhEo3t>;LLb` z7QaYov095ggr|yRMYi7;Ce%=)PXyz^imS)>3R-OaJUqO^mylW6Scu1eL}y~tVL2!0 z@yJ(T&X4=AsJQXdjF{pR;pRNOWEWoxAZ=OTBa~gfhHSJ)&QJzpKOz2tYp)pYPZf}L zS}C!S7i45uss(>R#gq!kGv%E5Z0unEBL;JzRo_%8x)!k-^`KvX z%p&`9ZDFz15=N_Hg@j;V{2x@t9nyl|p}+p-8+0#Patv=`-iR?ZtR>7}4kV}8Q5NEE zBMq#Y8}V*wew*3fVHG>uMyWS+p7FHB=N5T66WxwW$* zpr)Ri(;qV7a+78BTu^Q?U3l%S$IAjpY0Q{Waq8R-kDfRx9#EM4x;ThK8!yT-w!$V{D-LI05vLdZ0F9Q6TCqU0;tqM45T~P+(9RZQ1`QJ}{J?CCc4ust$+KzULQ#u-M%> zE#_T}aF&B?q+n?cK`y?|)O1jaE}d2FCi=s@h{>{tJ$7HLzBriqXQf4_K1BfhL`}0NQc@;P)juvRx_8d` z;|w9rpl2%jQjz8cGn46s_)37q&EQyxt(I_n_6r#K#dacLmX+`5xK>0qj=@;cC*Pk% zpdE4|w`87}wij{9QqXRgGUP3kZECXga3@OX3bBm{H3t+jlNGkCnPJllE6XgMLmZ@Y zldcx03o^?%$-st2bXI7zRk-P2V;K;avWI*=*_7UggO^_J1WG70?Y_?ud|nXmj@vay&oL2MELJ9 zzS6hU3LxZymWGjQTTw1OI*U?`wz8KXv!%zdV&CiowIr^cteAa{uJp*kdDXI z2GL0+-X;PO_x0n>aFr~~8Ps1C&Q>ohiBN=VP5fV>E|#1Y4aQ=t`V&^MF-&1Igok=` zs;fh1HC*qJmP3Zqv4>!%f!#e9;@=ejb9(tH=34_I8-h?0?j#cSFn5cS6}Ajy^Po2Y z?h)j=lQN4@sTK#U36@QUwn1bdzf+(y#qlIcvj+wEOZH>kj3%X6X)Q1q!7j2AYp)G~ z{LvyZWmiS{d(Nu4jF$l7a+9Dlb{fIIS>^KnCrKUFM7VFH&a?hH#K{E8we6aAW4Nyn zn(>96XnGtcz}$Cnb(3CziO|980-9Oro6y;o<3wQ|4${8bN<@5>30GBfG8>1EgrE?3 zx5pdQnW9`F$tEZ5hU)Qw;EV@OyK1pOOAavVp_Usa!1-4=c9BcASh^Xhos(gFsS9xG z4E$1^FcX&JW5I3}Xw|CGIT|_}Guff&p287Lh$g~%sS>{*ljq2(C6Zx>xM>q(b&FtF zIe#vU_Q4gZD2>g(VdLYAA1??)0PI9*`LBplW5>MwpmBsPXiTrwX;8s^F3_UT;}|1i z)$#fFjL3(HLbiHGCqA73;R>aX>foV+MdwdKoossUCp}y;|3pB`k9k%XsY}r>%xwl& zgL*{jcp;WlT1VH{gU&`gI-gF-&tD%XYebx1(krwM_)4fFavqYdlHa~T9+7PmR|H!% z>pw5lovQ=a6R{zrGi;fN_TpF36pz8Q9-I8Z1X|K+>?OhOJC#8fTpAP!^U8Zf>fqr>MH)u6H0BR=c(dtQCY8IcEx_=Fj*VjzIaJUUS51z)pZA9 z_N_pjHxnkx8~pMJa!zfG-1G-b-h^eXGTDER*cyOBq8q`hw+R2+E^CRnTmTipmEc%@ z8^SIM3yw0r6OI+)=#+#r)GGy8DmX~#^mWotnz5wj>WdZr_i#aG~E3n zq)|LQ%LfisydmCRhN$w-ZkH3je3nJZx|D4C4j0x&4 zBH^fo_gIcp7@-eHfw0XE{)4<@ABD*S|Bh}wc}cXCY4AhE>G zj@g9>E>_r*qrFLqsom`}Ld{ZvD-5_ktjDv$+&T+5w3F%nds#wN>`*S)g}==)X2BBn zn@xn#N_GqH3xUPaJQe<_!cGn|)8xYCv|WKH4y#MvRue%HTJdY0I@s9V02&iYW-39a zo+X%%(j~0%yF<8sJE)2+MY`2sO+v>MO>1MRR_l%_dqBC6J*_B!b&+GnZ!qLWI~>G#R0g8C`hw z4lq}0eUSNXwCKjf!Ojbj)#6HlPLJ&nHeE6+8;oZJyMUb)JH7l&TDaPvGKS-|OPHBbhh-UiNH+)^^m6a)EQS3AHY*UH~(Z+gm7*55X-Bx@NH9!-qChfxr*cTxFVdOf4!j>d(jc$&8~})T#^XWduUQxy z8w?=xw5AoS)9_~r?b_Cbw+ipWopCUbYa3M#1{N{u5GOX7{;(-yli>wOD zW-593WHp;s^%n_u^Wzw#v6g&AsCks_6-jP>Y#uN7Qdx=VLMdt$we4{VH|lgC=^kzr zL~RtSRgU)mPgGVLa>|sh`L`bjNZHYcaF^!zJO)0@87Oq{xQ9U3G{;3+Hl|I$;wJ!` zMV%tCh%O4riil0AtgB?}6h{h&$?j@7UdqBP<0eM&MB^-5kVa+V1un5nUl2y@u!>G4bI|+F3!m+e z(Nq><7ZDa-y-MD3xd8LV&Yat9e}#7Hyb{Be{&-S|Kk3!^UmST30Ia2H^eGYk7h_vY zdp`Uvpym2cO`yaG^(I5`IQ|kaS18J>a`avb z?M!&&QBC|zA^xh)D=TSS|F(&-LPApPp0~X$kB1vPY353ChA__?%$=9cfO1|M>|&!U zppgtNOJl0S@I6_c=57<=h*R)@63>Y?3+drzc!)2Wm{eBje81~T;<` zq(foLQT_#%ydKhQ75f={7PB;Kp3j2R{2!3+s5n&$v+nCMpBCS4i z+)bda@_LmwT1n__K``2Z_CoSm`DSPj*gn=b4=uL1nU3 z<~H;;{xDa%bEBz+qaJ~XrdIBXi?VRb*zxhY!sZMe?)VWg`>8Cxsf3Cb8btr{H)x@b&vJbgMCbwTjQyIFCw z!Y%+Fe)S+4ONoW<0?e8QY@nwZEyUT%&r_Tt+MH`Yl1^Ov?gld}B!%L=E(5;@g}Tu4 zir*ctH39U$w24#hAx73FTW~Td_KygsU4IA7?3^R!| zr$#|E<-}cW9H1~oVr=T?z9%E=Uxb*k9`t&Zc&iDdWm(LBzrSxqf77n8)24xU$6)p! zZ!6>Zlm9dVB~|`?4?tOX104OZbRGI2h)csoYdv0sTLNL0JEKjusT%la{wc<#y4rfK z5X*+kmUwdGakPhl%(oOx7=#OOQe35COQEYFQw72k25eAwipr9~)rlOfz9lHDQ9bp! z&?C?uW3We#KZ+z5ZUbt|-yQ7&oJ3L90Hk_c6Vg$LTJMX3TzKh!sJ0=Gf=~%9VdAPtYh-{DW>Ag4Xp+pC3En_!WpqV(paI7J}& zB&nvmIdpBF)d@`Xd#lt_(^f)Q-m!~Ic`zET<>WhcOdCWd%~0gNg*zu)5pg3gc0zYq_KCT-nbjU`@$ppMXOaszky zm!K$gE0g0j3R@)II3i*vyT{9UTuzF+MVYIXG2>$CS0POn zbWT#s=g^Jx2g&JT4ztCRB4LtdQjfV`hj58;bxREPc00TQ<~pfyNxs_MWyrbFzSJ15 zRU{X^9dm=YuK7_i##eWHlPDBf9v#+ti?F3AGmLutLx5AnR78)lyqF{8$w z3B|9B7Zd1Guq@TBlfn2QOS%j5(i7izHht`dE>+lZd$fB89}NqEN2#-Mc8GL^+%~wcihXdjbK}^ z$UIT%YN|y`fHU=vnbd9Z0X^5g>W5%vsiH?g{w`Xzq8wwce?vJ7&U&&rn`PYpIZ{sn z)7RKk6s4G|?)ATqAS_9CiDa#gu+GPhAqSSxCW0K;6q3V%qFhF9Wy%9=9F;|2Y^69i zi89kD_#CsTy01ysbTQ_$EHdi2 zO3eNrB3u4kt&;qQ2;p-Mn0O7bjz`fv2J=eyAkRI24r1<;Apmv!wjlG%G|?5_>LJg-vF)#v-B86$0mR}tXKC6{#mhbA+IeG8oRdK6mh{?=|H=&q)- zZah>-<5PJEp@G5l_kvwcy#@kskQN2w4g-Nq zB4ZVymYE)!uD0_ks33A{c;ebav^m7$LHdT-e*w|pLbm%UY*{iTQ7`ffTZ<$9=P0cC z;$nr($aKiBMP?#E6~|q?Ujdu}WzkNrI@`wXs4PWqSC}@f+m$SqL=l%qePk<4GhA&B zBgqOA4Gg!Xguq@I(n!7+Mx|>BA7d7QFeig@je>mk5M*J{2l132$2E~6vlIPtRuf^VJFaDX zY_e#cljcjk;1PQZBPTB9wc}P4!d;<_<&4G@&aOcXjF)r`7q6Ju=gQtWeK*@s=S z#G0{0{$Vo9(VgWhmdqpJH-MGP`vRQ3?p(xwL{r;ER02IqIb6oo+A_ll(`bt6J=MoV zHCnCpC}_Z!26W*DGB1J$3L>>uEW=+_*ww?FBsY2=3CTFd)}RmzE|W(o6tVCr#y3p_ z?*pov1(zj`vu5CA$)l!G5@;D=!;HBV3#w_q!Pqy}tIR-03o{StV7@2*C!|p$ZB!&? zdCN5;$x3Q4O3UZT_vnEhb*oE-`9s~k!Ov36_B$Z6rVDN~TUI9tbRl64PbYILR(34H z-7g+^@z!^2^J_bTQfv4@AT5!VQz4dK0m2;o`G{7y}%N5sGx8Ftdu~4Gyh_tOU@QXE}3|2Z&M>ggP1v2adRxSsB25 zQR?_vq&tl6kRvZ{W3|TGs{mU%_I;xD;xz*Osa)ROCz4hsm!5_AUX(NH9@M&E?NxJt zXD+DE0|mGgG;y}44EQIk2IeF&dy0ESrcncJ&>vTaaDnuY9|t0u_-JM|-g9jIu$7|)uaUfWYLq$6|*~u1o1gaEQ3UrlN zAxJ`kEH_Od`0Yu{g6{;Y32^2WE3FM-R?(=GV=d86oBbEI)f6DE6zGQ0Q$pHh@ zhw4;aU0p3g%SF;ruKrUfM)bIeQ=+D@YmNewC3^lf3IHW1{Qj+g%t<6w*_~mmB9OwQ zSBNiP6T)ScdngiFer%D*ROE1MBuaoyA1B{`5Xj6k@ud zD6%+5;qtg37oiL0q?Ka(wZZb_z&1>0y9f_&uT#WCoUcvSBsTAQV6G-dLyVB^1+XGk zJL4P$Dfo0e6JT6hL_}LpPmc8goNovB@#L;F_7g@A?8O)C7v*;}=BT{ruh(u+BqHZ5 zqp{Y8MI!3MD6qIzE{tqDSwG@!5qbQ#ca6jcqG?#2)U!ibdn3r)6ZBYb+$6vaUqkcq z7m?z9g}HF{jUilOZN6bYmsN#10an8e!jbz|RPq|Gbi!+;80^ z$R(^`l@e{66bW?J2i>P^Bct^hi_tyBIk%0RHuK-?!}{WgOt6AXh~qNRkg4tuHYb{E&Fw37 zjNODdCk*=JFFWoLL`itrtR9b+e=xgsp!Aq^3qI!#pW&}lYcZ#aCb3SG>09kU*j1Mw zU_Ugttw6UnkG#c6BHgt(tYG&w>y7}F4L@u&>vU0$J%k5=w?(=D^6Z9^KmMKLr)%2r z!<~rWYV#tRr2qAWn7drYW!NxX{i?)dGLcRwe=RK64Zr*nS&k^7OC$L%hBVL>~SqY?|?o0LMp!l?msIdnXRd@Y2A?m+1@=9h#u<#xxm3i{6xR<_YtY7fVgCxQQbO4lu5veLR_gTLy0%gN`R?| zd;KTuOANO!1H*Q?6DnZjgp?XvDol}(j(I4dt%%%-*QA>K;XT<7=87O`N}@n4Odu)f zRoJH$!6RXSdHkxXNxiz-&O^8&UFIPy2&l!oH4y&oK~cS62Vs}Lljh^Rv;;;*CG{LC zpb*A&+9|R0JfUW|&Ud6&x?PZq!G(xet`m}6q$a#2hsRACAJ645uR+ISUBsd8>AAXf z>c#&Z%$6I7G~eXeKDnA)wA=B~#tlYAL~e7;vUIflmLP9LcFo1sT}KOcQfZK%P}1lcxvo~yXshre(N|=} zgu=4DivTw#`i0!Ft`_J<8j-GXD8@HHnC$Qy#ay~$i;`pBI7?KkYje_{UQ^j%05JDN zBr0w~3UHbT$vStxT8L}O`$@^ib%MyI*`uA$@`FS$&(n?{$!6@%exA6{gNp>ie4ePQzG>DV3_oRAU9WMKN84XQ=!vnqnrzhaOt_~ zEn^F9>bDb0f%HTY{onwRR2Q@LcqsoZb$L`q91lSoWc7xj{^z2SZpSB%VxR|7fbI|vQ3L81nF-C@OVxH6=nOJRBK!R8roU+ zRpjNjR{+VQe2bTVL%5g@+Up3&^2YL`gMiJ!+BD8BQbOYe=AtA|>#>s%7e(po!R*6Ps|fdWG?$f_d^B{ojrijK8K$~~Q*@E0jIZ24$iFEZCs)#Rl3vgqj_G&4}n(gGGKN9cuW9)aWkh&+P-;@ho-$a&52HhU7AJ( zJIa_*0=$38y#KlYGXhmKF6K}0-{U4th+}_C7>=?3Lq{*d>_|o~dXn$H5|c$hdb#Em zVQx+)6$vaICxYZ2*3Y^?nYTR&$hp!-Fkli*PRM3l{}Wg}8D%y>%Z3rEMm1uK;nW=O|D$*^)X|wm!p|1`kPY~+T;zCEleq8_z)TNA=^(=p7ZlFIhe19l_%T`lBrq$i)-m&mv}e=!CV9b6H(DS@~?fxHvjn;_@eV%7iI*UAXE0 z9fa%0n?R`Eur(57*63ibk9#CS@(^TfH{x6YIa$`mqad)iFP^qP8|G33Y`U^&Z{3xx9?0XWFC^!j}E;3y9~Q%GpL=JR=$wqN4AN zw?$BtG-t=67eY~0j)LR23g@26)ZE^c6sBhhbz*%ya2cQHKR6wavo7LWOYxmwb7v)J z#(kIY!OURVU;=ue1mq{P60ZnwQ=_?Mmrc=rd?^^^O;)SaqT8tW;7VnGZBU-(B%zib5J53U*(=NOj z4Z)cUFoC|*jK_tUcQ{dH;C?57gwp+k*{^}fC6nt|*!EjN=9HXQppvGqoGg$!_YU{O zSqkS|;BDGeob!hQ!+npnWp@k91E$iHHN_Iwf;j0mU1v{609#!L=mIwNh*6v*pw(Mp zU^)nm^Mz7^5$@W|bvPhn{(#tC-H>E@8lHr5qt zPB$fYY$@9HvqVOM`{y`mI-vP(6-}Y#mW?H-J(jxxz(pEvbVweaD8$K5??PSGCOVCj zVa{4RG9_7SB#sv4g6W11x)M}_vDF{_2XR)I5<7|VWTm#%Rjy=iv5Ro$tgDEWWnUq* zFj|ggoFpQ5G&JgIih8%i(l$|+X>B(W&&6fJVSW|u|L}QT(?za0yTmZtlWx8mj|UD9=iq5 z1(cCz{O`}uq?w)!Nu6-ZTaCakuHdGYA0nK_xJgN@S^KU6Nf&}f<3J%)WjgcDL*>`$SWRGM z)c8vgzcwx1AAd@S>!1@G#0p)8*zIn|E3A*=PeRmn5*y29It}HuG?Va#=Cd}1I%gbnt3Q=u&uQv`7Vdm2%(|U1e2{A>> zr(zr}h&*+2$z*VbNT;dc%3k_wA-nU*-{pQVcPYd<-p^ql8`lb?xXdx}^9KmG z+Ci@u|3d)%4aI2EcXvGq?Nl{E(=(cJt}yq>e$Kh#bJ3YgS%)Y|YKOl9J9EYi(}gq~ zlO6(eh69yRR-^p|kQnyrX(H}?7}^zUSn|!ON&HJ;PP{Fd z;;i-v0DQ&?c{6r=)L)BBP%jAJD;80l0zOv!%eIVl`BMRI-F~<`iBw4E3R4UW?W(0g+9S4Ki!qexf4W{gSZc!7?D5MGb} ziFU!e`qeAec&3QJk#RLX`8zbcv3*wj(&JvQi*}!3 zoU~j4#211+7UYz#*@6GA=Yi<1IK_(JDC}fWDlzmf7vL(h*6QXyPMBT*b7|nRw2Gey z$*I;j2SdF3Mi4dNCarqTUN4sEFf$}qEf)&&Up^60A3xMesnR#gePDT$i_h1RMvLLUb%qN;6wcHNHAIpDPELb=9e2s{ici0Qi z)MpD}4>?ke84Bic-tw~w@0NTW*d@Xah-b?iB|xKLyLjUp#PCpB0sFLl-h`yPaV;a6 zsNNvj=~8F7&k+H8@H++PR^x85n7hqe1td=zvSd5Bgz$-+Yy%$=fs z%bOxxSe^Xw%Jx41Tw}xw-D2HZNY-I!Gtjz-H)AFoqp(80TKMfEg(@d*v|5i43ee$q z1+N#GmmX=@>sZCtV)z}<+)PQkG!)YWx!|~ai-X>UbXjpuiuURS0WhwWM9+KBx$GR9 z(tZi?FX)>X@AoT8P9xn8-|wGJ1973fAN!8mM7tQsZ|U%U&i4z5i-;&Su_+PcX#=*S zW0xJw5B>$1I}7g>!pymZz-qJ)6>P;JTpPVoGC`y>vPJ`U(H|5EjH0$gi}#`svr12Q z;F6ESn%L*VBBi!Ic`+zv72pZocBrb8;7;}yvnBqQe}ofAlg!hOUnit zYt7G~nGZN=K|{emboDcY5l2ckjmw)vn^y=NsFk=KDaf^uDVDr3RAYgE10r3^pKgEL zC`t=EI;7VWZjFHnvHs_fP7q-}sn)83=ozF&e?rgyuQE=ulc$CME68&fx;#G_{;U6h zr96=`;to;dInv)B%YH%FnP4EAyeo|eGGDn5uJLw&kesB%a2-8-F3b~vN`sp;eTnaX zS->cZ(6(A{GP~fM4R3VFJ9w`k_wuO~u9!q~rZ{Lt*c``kf+*=!E&;CozanJ`bZnI5 z%sX}y?55&ELtFewbgM#k3v-7c7r&2_y>yW9xG-0ejk-K1%>6ZpxsL&2>?hL2V zP7#oMusqpaDB3A#g=fZI;2RK4S`ECMz9|BhOPm}u{}$J@h8 zeh=Cc15c_^n6zwPFVw7%)v{ib{iC2(k3+@Rj7NpIo)UTJ?W3{o_X`XyvP>eH@pEBj z8CD_e>9!L<={n?N;x!R@$(GeBM|_ld^6bSw*r1TeHYL#H5D7;jKOlz7!wY^gLYpqc zDd`MWyC&kDQpS^l3t{ITAuhKKR%x?M<^W3!!|-Y=5l*N%+}@57K%&e+Lj4;Zp*LBG z^v~(vaCSH;hD6YtIDBA1d`(p54jm(j1Ww zL=df(t~U|oh#waJjnvcY<4lpa2+XN-zksI}<^iBrVT&I#6sB;Tgd@1Um#NWy_A~Ke zrdYZ<+oC=%F<}LDnj9yIaChN=Z7|LjZLTxlNpt$S5SKPx8=@e6KLW~fhYUfO3N-m1 zBA6o7>(YuIQAX0{O|qfp`SS!j1CD968eX4&BPHgpRp;Y7rAz0ATZmvfpe>|y_C+OB?6i&&>MHB)dYm{>u4D6msjVMvB_dm#OM~c?2(2G7 z&0sty+J)BJzcN2~cM%{Ot*tKaT8l3RMRCSWSUaXEOmTSLwGpRgLWWrJx`@mwuA4Pt zfyDt_EG))&8cltAIf3d9I;4!JM7i?$$qkm~CBV!u96E| zZeT#i!qFLr33Rd8<>3@%D2@~4v^vn5JS@_k8l!TIaAVzNiht-W+}az6i9acz?%_5j zJ$!AOyH4>h0& zr^q>fe_SY9DXu;--pJn&3b`QG8|$uA#KnozKx`zMxcE_uLlriQ=`A?Ks>YuLX+mS$ zJ3Vd}We&3}>$vzOA$efyQ5R`Ez7(cCS7#hNh9Ifq?JaIqIQQ4&&-l?2;3^BJ59hB; z4CjxHU7E}95#*lN$!08H=$N<)DCMwYY4wgRg_s50+QR-#G#O808h%P)8UxL^USIi7 z85P5cwPNd4{h4{W%6qWqi*zxO3z*BU6hP8!&!nA?r$obg&gm=ho(OYO^wwX}{Knvr{Pb@r9(= z^rw(&?THiOTZNs^0OrmZVl2HTNN!W+DyD&$B#0lhKJ1~e`G!`W2V+W_2tq5~(XJjRQn z>PXp9SGYB-5M)%hsrh@e(jK2F?8LO$(LVC@N&a02iI3%QKNl*0zE+VG>%Kjn?I6s* z+fp3wT7n8aM%o0WPX37S++}Loo$4$#f#wGn^TxLqo6}O^T zW4PNUNb`eM+V!VEsxZ(nc_D$FHhMP#1ml{rC_kZVx&ctC_J)S%JEtm|(G zM-yTj5iTr_b2aIlC&cxUW4dNsFMt9do}goTB~uLO$?HVVdd2S&C|Q5s6GCj}6Evqc ztPe%j$X!~*hBkmSBjh}YOEXEi^)JD#iJ2pr^u${NU8|}Tjj&?IhQw2?^uFOoB3gZ% zqQ`GH0&rD&?FVY{jX`KttOH|nh0O%~Z^xM;9l1w-4x4eA5a)*^Q_%)?#?OH0CCDMk z+HI~){1aY7x{w%X*{g))aU^TPct?Q75teOO^A}&a7%(3F?WP5QqaKbPZxY~QV7;~$ z((3m&1EI{kg&8j@>>{<}*afv;Y`l2^F^jipX4_m49H&K^IENEuULs&%Xru|evIU?A zH2Pp{G-UA^Ckllnv<tE}^G1L{$Mr$43+=GlXVxl#-h zO#(zrVE(-y8I4m%Mqfgm51QHGUe3I3(ciFiYDaJUHHYz|7ISV3o+km>qbw%cb>PJx zj4W0Z;G$3t?ID{nW;-w!h4m`GWxdY!1*naHl_(ntG?UP#skdw`B&RprY){*-gn39< zW6t3%8o)k9ZB;`O>1f;|NIip<@p*+^3LGYN4Plqn7EA8{==z{<9*8@3gl=_C`VSWS z0JJduxf-AT0>U}y=9UhLSKk@P<&yzg^00ZJFh|2+k#W$7tIKGdU2uB}+4sg0V~gT* z*uMxPyEJc{w~GmOB3X53g}sCO*(eJPCOWX>E%>;_@x@}_4E zZ#K6IapAR0;zVgEKHMG5g@+^9%vWN$J%G$P4j0(dG5BMQKywbMWlHQSnvp?MsHfc` z(mib|D^k{4d@R(7!znsQXZ&JMAUCG;9+)=tYJYj4UB%{8Uyco z+%4KwLve^xop^PxB3a6u-qp~C4 z7o*b4^Q#Azky!BCtiJkh8HvYur^L^HRb+`_mmaa;&1_*gOH3WoRz6-4NMo_cjEh6| zfhZbFew=<+BFuZ(?Zylt<|ZwbHrwY3$_eXnDYnu)i&XyEUKwUFjXQ+6b@Z&KlsL=- z2kr|>0T?aSxLQOmKxIU}YwpRP@nXQ`-U^?+Ht03_!$tBdp2-JeY#W3JkuFSNaxK7- z^=j?!%kKym+JFi$Uokwa#Gk9s&YLsezRI*%yjG?_cFUGzFSe;Gh%_T`B+Pq|JW3^q z0wI&KsB}P>D_FM4DHQVE>Yae@c-gyOa`9XhY}POVO+|Q-ZQP?m^IQ#EaVOCQK>xH7 zcl|n)n{KV>7U6^ul68JZ&%-U2V0S$(QS&@99*U0X26JXK0(qsi{dl1+Zl#?IjcJZQ zXC_b*=YXz}{6U}@J8s;>n7;>-jZS-KbSP++Ab6*d{hA;OiB!Z>8odx^Ii5G$Vjj`V zl)YGJj8)hsRJx^GXAyP5w&>%76UAQvd&gSrEy(F`dt)>`gh+{f?Z zuQAddW}k4azB^wgmGT4SwsiH_ zLx4NC=3ka+=C&?@B%T}<$A~CruG@-CASVfNm6HbWEYU7L4;auDk?CgNzrY5t2IxWz zGe$A-%ikZXC_)}q)wC!{ge^hLa4=()B(drN#N;>K2aIb)P$hc?VCX-8paF!(-hMq( z^p*gR7sghjT5Wl4h40&>?upy&Z7jnwK3&_H8TW=-%{1zQT`$u0jKkxK=MLDVYXQ5 zFen#O>SAdGw=N+@JDxc3!6is7;7jH_doIA!2F{1$5|L&P?;%Ucu;LK~f)ZC7>1~9# z{KJjmyyu_oNI*9Rd2u@~(~LM>d5dL_D!!v-!@;R&=SvgEdLpT9j}#TdBHSfW3~~HV z$9S~F`q=5{0kC*pn0bay1^vu=M>`d^0E6*CCi?W=)WOG;QE7_)_~EeyAm?QD*jRvT z&Y3W)ee5NKBI24z4Kn$-A_7u|c3am7$@%xzdFtya0l5vZudP>cDRcr*9$tuF>>b+U zY+;UuKa}LZV};)WnWr|zkji7sNku%Vt8@~&lc3CIESKUN(eRBIni_G)$;HQJjXMy> ziDnjJXBRgr>o6bV0y@r1&zOdU_3axpyL*+A|!Jrxb`I7f&Jh{*+|Npi?TWxTG+ zuuhg|JBN5VGx}Rn7Of%71(G3P)Xs&5J4}x~ZE=z)%8p~d-grucGnR^x&Ns&l5I4n? zu`9*`B3)Se28yRj?0udSkn_AaS_ILVJtS9NBvSRn$H4US9oE&?+Z3hgk3RJG*w4QRf$hTm~+WvGYeGM`amnOEn4BR7(gP)-x6eO5f2&CRM z`UVb8E{AflFwsiZ!W)-RbAT~GNsqsxh=rj>x_q&~l|Zl(NiNn^7+xUlS7M+1p;76M zHicb){P-`j-RZ4Axi$9MRYY(P=|%GY{?$&Dw=Cj9h25Chr>sWL3qms+c}QN+9=`{2 z%M7s^#*w0(X7XTh4WznDx^4eIQEq2BG{FMwTOm%7xe!O~opH$@3P|?nXnf*BK~%R3 z_2KWYB}{cWU8}~wM0oz;Y>hSkqU#*92QBJL3KO#jktIGC;jFPgl6lACHx&OOKH=JW zj}l;+(WPDi>SY`U#3drkO)38^KOgW%K(i@%-scV_x{OPNng_aQ@rFqEPpM;Y`_qU; zZv^vfmdq${h=5sgOcw0=G#eN+#q$DkvHMwMSU~;+;&Nakz`Hedy2yaQ%tmsM4e5v= z|Jgmv1u5Q0EfG`_Tmg((ZX!-=Sh419B-&{Yqf4sGA5|P8FgH3)j3@RuQxFx@9L&hS zxlBgJtnr>`_{Iv=6OCIST%|Vp40rYZ3}ANf7=Z20-Vl}xiyv&Jqg8GNF@usRlsrIF zhaSQ7gkfGFJWOGie%$1>C&clh&31(QxF|WpG7**+;0^hc#CXeJJ(6SzyunZESp-SgDs- zawb7mcM4O#Mlg)KGr_&{Mtmf~9PDDz?TiU`7Qocx53eGi;KbPY-Awcp-V^>xv^mqm zQ`Fns1({2aB3o7+NSp@?WvH^wq1ZS@l#31P2h-z7SLPT@s+?t>em60kDKqX+JTDqv zwK0`^t*|*EqXw^ybKC=Hmb1L~HQ1z27Q{a&DRD1OnEcaLF|I5lx7@6}DhLj7vOYcj zEy69zS)?Y@!F&BbE}rOA|Y*K7U|7WrqA%6j>r>#s3t}e48?E?U;N&v^lMH zXq~u8fEhQ6{lhIHDFJGl{&+})tAon1jzi}7$pb(x6h>JxvY9M|KM>k`V)sn&z;Gq@ z%S4hscYUx(tF;L~N01u{A9|dvqz(HeLS5qN>Rp$WKUUg!kpn(FTK>oxW)kcflk@(S zAE^mu)UwQ)Br>-JV_Mz+4}#p{$+@n)UGDOb|Kb9qgn}1Ex_-!vb@WZMJq$wqa6B{^ z^NYx1iux*n(*&8LddQ$R?i0{rk!09d<&h!^X5}19tRcWrm|!F#jtFTLExWt8SD2dy z54`b3`4esd^ziOMj}k!*joM|i$BHCz;HyKhy@j}7a^OE2_Y0u=VW`GXzVGqk2P_ce zwtB-S3WPyfr6(OwA}BnyC%#088Gt!40%0}2|F;6d2Mvd9JW?;nC7;M^Ty>FdXl@>G z_>o=;I9aG0y3WJCaYO#8y^9?XZ^%AbBr9Ex^y_B|aMZ8ZQggfI|`S zfx^xL`!DoMPeFUoO7h3V3fXV2rvYhTj+h4HYY}9TCT;FWJyRsWG1)*2izaS*f9CE? z@ZNPLz7gRDd!N zhh*_fAdDuYagan_5k+Ybn7DxZLJ?h;cUd{BvEYkf?gHGS(EKfc|CwV+?Fdo)ujiLy z_Lm^cO1A!N{(d?F|TbyFs>#bBpv4 z;O$ft57k&kP@Xkpi$2YWw`aJzmBuwYyIxaKT|UkQY4$S&kQD+V{RMcS*4 z1iAXGxYJ_G{}%<5TkISKnXl}?v07ng5)TNZ-aXvxKKm_)Yu)`=;1hQ78~6;VA^&_L zhzg>E6;Hk_V=z828mY#2C6v|yCgJ8koMt!IbIw)Rm10&vgncc4tYXyut-@3gP2o`N z_;wMqo1=Q3BYp=!Jz7t##08>Ull1sf>LK?Dbf@T{$URzh{w0h=la2Y3??SjrQX9(y z^ooKq*SKvQx0O-)aTcoaw&;6AfkC4P(wpb7u3a3Wuv25)CAn#Ze-@CR?Q2G#APRsF zww}0NgxkL=uNqh{zYpU4xr^5s3;e5~QM;}cJC!s!)Z$ptPC*xNXq2age3=2)X?lipCB_4vp*C$@`KVZupC+Nuo+`N1al#~hOw2qMRe}Q zS`)ajAjE&A?Tord!kj!7OWDwM*N=*%*j%>96G(Urp zI$t45Oyzhv#(YvF(d-&#ldL`M8UkHJOuCR#VnZPwopNY*oNu6?9WJ-6i;!z}#9quc?!rk4vbEn+BaJuKgcknd?;A>XiG1x%DL|F;Zn<`hZZU zhg8+4_e$FS3j_m(IQTkJVRK70fqyrD0ip$9P@2wLMfrD`yQy{W5|r~ss>M97H|`ad zdmg39PGTP|f5ufOqQGE0DaiRHHD5DU`4WT{WxDT(HNJ)*^Yq4493{$`bL^7bU%gR6 zotzMHz8BvK%EJJY12%%+0=R%=iOQX$kc2rY-Fed;`iB+GkZppSvFt3HB+hj8^+mVB znm_8uxnGKKv9(8~TOK;AqvAC?{-AJ9iA7I)+m}j6!mOmNR|Gi?1W+Nb2_Xx;l@wEE zGrF%$*do8QnhsGV?g4#17zg0qPt&Why^%c#oRrp8}i!dtN4f9`qGNNwC3ZAv|#o zC~73dX`Ca%D<=v`8DUV`X51y%b(4dk)U1={1oIF@eHD9(bg6XnvOkV50Ty*G5nWaS zbV@qb*tz({++||Y^DMUsX)WBzwlqzLPYO0?5Gpb8)+F;*85aw(>{P88U(8KhDmT;&kae7p-b-=5tpbe&t>z$jQ+Gxcqp$vGL!gJv{){?^05m*xZ}u8 zI8z~ljW9EvyZ${fU9@Y$DkJgXN+D*k&nhKS?z>=-l+J-U^&O2qVJ;^p3@;4jOsh5BG}Z81F96{dr>L>eB95*$=4C)p zd>QT;jwKX!F|;k#JuYf-qEOn9`y9P2LdgD$0=R*6(;J7ev6P_P@^S}Mjg5pbtfuy3 zqE(HD%YXEsS)CKIMHefQ!E&vuie<)8<(x@n`58Swl8$_5dwO($i^A^=IAWlZ!`1TcPmb1lY22ZD= zb)l}6jwj-W%R;MHqnxQkMU)d|<1~o3j8lZTg81L6sm16236RUr{Kw{aaRIqsF!5=r ztVO#(mu7%F7{`cIHzChs6?QK2DMNoeS%`DNmI>zxqcMK@0zv@dN?uo-D#-cNxN1cI zW?nf%APJ9PSF!F2gkg{#o{MutxTffdQGDl*D7`S%Tak}uj?5#Ffiy6?S1M3MC6?Kw zl253MD_?gM_k6`8ALO&4SZ@r32e5A5q2lDFY+)`beMc;zqaG!Yzp#UVA0k|1^pV)7 z;*dv>3(Pwf?J<63X!BXJWi7Uv3E*k7uILuvlA&H0;av1yA+8rznQXw)yRYx80!q$! z>SkVDwfKomi*C}-vl@tN!2FF41Z-Jh{8DGD6k|oWuvLUSsV)yF|3RGN?Qo83t`J6( zv)F61e^VJ{c!09V*Ts`%N-SP_uOprkL{9jR=FyAQA!yACHxAcRnEGQk&cw+)y7W(- zzTNuA#v+L;Rm%dj8MKI24FGoOcOulO;pW#l}KAZxk1DQbK3AOJIw8z6iGPI3O2cxRDRe zt1&DzHz&86I+}5F8P5(T($IQWXg1u@fpzOinR&(+|5xit7rulL=QB|6?Ueg47RD8Y z!g0NU6|0GG6_85WV!iy4Ii@Q%$pl*Sdh9WQn63wlgUs@xSCD%SJN;U6eXznrFf$UZ z-{jR}iU8Va7oBz_-~-X#y!pz;K>r-5b?A~ z`)kwd8gdkq*63!utuHPQa{`wLW7D-koMp13VWt}rGs#s z?@{_sfvyOhQWE~yjX+!xiqWnu;O>Il_?&F9&@HtwK$41i2{?8ZMMq4#fqg~zcl1oM zSdQa!Y$Tm(Uu?7q5pu6)wu-Gvx>GaR<)U3o-7WVH?@57V!Et9j-V@OpG!w?I9^VR} zo3*1?q zcor2-iP(=LD~oW6WI4M|%)2=NCF(=Xx1z$i8G46f?@ZdGVeBW8bhtT%x@ijtPeIx( zaTY8=@4FJ^&RPRU8k%2YOXB91Lxjh4z9Ga#^#y=Bul^x--7K8K;1NGw%3+39oUv7r z+tfNYO*P!u;R$uRRjf!^-ZtGD#I@rkJ=U|X3`jvF8;UqofHj#&2+ZZ&I!S}~4Sg_w zIhc|E%&wSqn<4||B`ui?3ULOS%&PHh83l&p#3()%qM3v<=yBT?3CQ&uBW*~C6X0e7 zv(pCxB&m(k(c_lwpwvOJ+>J*>@QL*t-mJcT@h2}9Qk97UTw)EPB(_(C@jp9jJWuQZ zVg4kSHqBU3h`T(yHJpD|VlP3AN*vU_R(`ccyb}BG$XDmVhQxWCEUdNA;#n5{As8;Q z(@aTo(TP4%7v#DQrAw)&{{n#0>FPrKSA;wKAX9M6w{sDj69g;(Y62*}9fe3hnOX<2;3(1~MW$gLegZ zn~1lx$WMZ5VC zj?v4;J3{h=$yP^doo)62a@qR&(V9h<0P|I5h+0}=KVhEeFr~m0L1y633Z$s`OsL0m zBB+sv@vVEB;#>(m%RGCT>c<`(mANRVPo=Eg>JaAilLz1AmF#ALG=wheFiu_&N$oH( z!J|SgcGw%*wZrw8u0tLpq*Y2?>yvbTia<))Sz+=!Qv{9Dfo}cvl8LB7=?%gsP zk3VSB#9?3;(OfAWOw)~az!e0#22-Wkyg{_nqR>qg!0|#ID0K)#s`a`d%uduj$pZta z>>)IFEoz3Q=4v5Mx7mxe>pnfuPM3!<(5Z9bUXTYR=|~aEf;MW{X<@^x5qDT`krors zxOhdh8H>YIEcO=b1E7O(*MUiVb5Uf?twTJ6U`Z^xm0%jedo@7lLZkSqJzZgclYO2XyEu|!Kt8> z4|PX->>$GIZL6TPJ6tpkkezkI$=igubkcRmGUkq)Lf24VZ~S?L@AR-XvcFwnHy`tY z2Hp3jfw%~8lR=y=CT<~gra2v5Aoj2VT@-xU48^^ob4xH}G(7Gb1@buJNDQ9};&GLG z5CuHY9Hq&b=P1qog7d7DOv*1V0qn->J@Nhhi!AhZrLLN6ASgE&*K&|X*ntT1>@S^0 zqm#v4J?X6ScmC}|qaF1Fi{B;o!{VVhO_(c^-aBN1y+DxKfL&>PBqCS5A2rxlWg6M_ zc{Aq!HF0QImIAgaD~ocEtW+w|SCRw$wK!EIiPMvKn(8-2;v8SIj)*29x)7u5KcZYT zY;!c^;s*x-xeXcVHB@Bt3ZaE%WVorqc~Tsu3vy!-M*ODy2U%067&C;JjnWWcZ^A2( z!pICS*=ZZek536=8X3E0{OMqVd02V zFBU!wB6laQitBi;W$jYUXo2b3*KRI5hl8?p|Z| zBcaG zRjnQI{gc3)f3pi2g@e`-VE}PTA1j^ggrwbg#l<_#39OhK-O(C9V>TH(|I}=i4gM%78RfuvTQ@Op8 zukx&PR`JUKGRJ-*&F2o5YutvOozq|j?v3q4QRXfVFRoOWsL8CFTT(IRcK~jIbl|Ej z^^?NP9j@RrJv8HMLDW^xuf#Fu6#oo2BuVnmKTFIz2(UNLEz-eQpH{&E_>2N#-cCn7 z&k7=G)~$N1ejbFYg4&89w1EH$&&zBvsxUE;#p`j22(zsR;r8nM4aZlNxUT%hL!j}3 zNUI#Mbi}FEfb^NH5ViC9kSi~RuMAsm7iNy@QLrJd?ot02h35Vx6P={WIP*e47nTDj zwnR4waIBc3(Y4gPqe;GNBmO-x_2@t z+TxlrQuZH*^zyPmc-5%k5aklWZbS7xrl0sgkkdfR+Yw10o1%^ zLeyOX>XkxH}x8&z;gm4vk%qE_^ z36f!@>m#*T>So83uhUpdgxRfUTWw@8B+zW_t8}oZ8WCVFGC6UBn8zl|A7w*}luGA6 zQJ`yv;)Ko>X9{u6IO-hiXX=kjgt>IcJ@I8p%HH?JTlh+05#(rw*>8n%QJOU@NMlX` z{uw`gafwLhs3D0A@PrW8hd1T=?Wx z9+5S^yA6!|HG?6oo^l7I^XKSR9p^zI?oWDd2T?uU+L>}EFxBGPHsZp$qWl~3fSmR$ zcNd83HNbt#jYZP7XwN#MQ-sUWkJqIEos+cv#SyqC-V^(aaGiBN;>P%mz}y(gT~wSY z#Oa_=^Va%BVQyOa|CTO)vAcm<0%bcF4y@x|MX=e0*VJ0pddjotp%_$}G@j;h3I+z9azFOEHEW-UCoA(NUIF_QSR1gTOg|PE56kRRuYJ zgsn&;IB@F-f1Cm&}GZaGtT@ciM+EH^C0*GPPkRk2Jvk)#h$2+LT z9~O|ShZz_%Y&GV3&VO(`R*79in70kRZ`Gz}lj1O;E^pUxJ#H51Zi972CGN^8px>>< z^CB{H*!k%sY~|;HoGaVEv=@)9g=u<_3FjV};07f2qas{rylrD@wZ#hsz_G`K)niuy zt+Q)&0pu7C;mB#;7x_X#En~-Ykz~SwStBOD1mRNf6c^1+-*~pfSk!z%__;6|svVP} z#a|}u!AbYk0=JY9O6S*jndgoa;GTrmXh`N5UAo+K-YbsC=#wA2W;`gsX>r;)84pB@ zz6z3upQg9{h+G_uTPem#NqlZ!0WM6tcZZ z&)aS%P+h2w>C?_4oHA+_8vi5#&YD$kDCU3LvC)U3hZEu0xVNL&Zwa8-EE^R2r|&@J zfijGuljPPEln0hxwBAItIn8$|&@L-u;1F&g{vetplV|gIx%`U54K}~;@|6;#ZK#xO z8wkifjN3Hyp;~M&%xxp3gYKP@(XK*Wl>Q1AZ;uk~(&`4ArkWE3xqb+0h}zc+aAzDq zt2X^TNCpypJ(m7w@k#e6Hx&t2`*5QZ11&<2k?t=dH>Qq2cYYt*nY5+Jw<>^Cr*n(+ zG=*Iq^=|Bo^@Iahs}TM1evYoE8K(S;PfnYgBIDvD(G~=xrDAKC)|$rzo9mc*YMS^) zh3>{UV-tg*Wp zia!gYVtPeGO1{5|hA$k(YNQQ+4#^(w(Lqoqc>Qn#7pw(Wi8NQG7+<>yL9plemm4&s7)(sBgs|GokCHanP51am^8& zSS%L!3cy{i4o{gwR~16G7~-m2v88A=0Rm9B!fpbDc={g?yT@Sufns+twBu0GxfkR5 zfIY%vLMV^o$EN=)K652X?+INh#D&r&$!1LX8UQYGrl3LaOOgKG#l;G4ObT#cVQT82 zlrlWKzC_z_k+IjTbBGLf+0oh^QDkuy0Y&NB_Q>o_p4mkDT@$oZ-$vGwQ7ZI)%YZYq2t!DXo zD`O7Jumh(DRT@Sqi19G$w*MpkN9hK^_#av}PO2~w=K8SBhbMCIwlHZ}{}2yycLI6h5^sojLZ6b^G3 zO(NX=u#!nH9-l49jfO=p@5qX#OHIVyck6i_9%RwIT4BfIqG?}zEZURw`0-<7;*TKB zTExKd>%?>c{*JGWSYSR#*Ne6j&#s#vgp80<;sJ%7I49rOU@TD3J-TXqh3GsrNb7)$ zkB+!_LBQO2T-@a*ZfCqIEO#-kG$CBBwGfC4Hh|fFC4MWwQP@xMT2L&%a1o`epJd{9 z0&-nzG|^k4%{FFud0`KnUbmnQgl7an!Md8P;_ zi2o;Yct?PnV+6Ci_eE;$!1x`D#WHr=C3Y*;BH$VFG%vs5_)D~hggk0%K+wIf6KZ~7$fw8Ye!istlW3vEe>c&bWw!Oi2@1Lt zeZ$u0V%Q}RW*}F`7Ye&wFv`&hX?$D$!}#szIs>cD8%q(-m2York3E-$bVKR%KAkG| zl$ajXO-sYEKPFyU6M5())?NlmO@gIRJR>5{HYVlfNCtaJcK#M>?j#_h7gLyWgA{ee?%0K z+34|D?FtZ1xef334~V2qP+TW}`dh6C?VNPOmdms=0qMTO*#caz^tv?6+jk|W$>fOQ zAod+oq`}QXn)E>d{-qn2%qZUDD?whZco|k+{dDIf=3Ke_m4Vk2V+7Dgk|6JMK25Je zsSzH#D)EO*pp55;UxYWMx*pFgQs-huXZ%OFi-1;?WxpCHtl~tb>x^yoRUtf{u?LzG z3yUTPSWu0L{o&j$D!mLOwK~rD^tiGxHa21XNrjuLdvvwNSMcg%!`;mnd|3Y9Z>T$@0V|` za2}b%gB+cY`6)nV2(NyK`R599BKRCaW3P?VVru~AiH>8w7$@2ZVL*lze*(-LOi^T) zGf7ceZOinNPwx+yDJ8ajaf)b{Af3lhf;-E^dCL^-(lavwh!|MEuUQm_wXQ#&7oCfP zLZuO#k2Szhw_W_);{fRJ01ckAK0%}_DI;d;O$R++V9}iT1YK=>0q{gH)7?V?;AM5X z?rIPw#X2tvvvYq)Bwd9Qm3llT!j+OGryNJGGZBcv$bBKY?kl3qE}l@(sp;~QK-}_N z>+MYjG{*>YG8h6eF(QizazoTHs)!dvyUfhc)O>--0Ir$rVYD$Fjm=9CJ8;><(Lh@Y zbdJa->=j5pVQw@OFiB~yW1IxLxU%-)=QH&)wU>bohoZ|&fu#0&|2Q@g(X#9A8^|*< zwVtLAu1_-4U__oS2#)p-48($KJCPxEpG8Gz;i;>it|ThgiWZe)WVR6kogCdXox`3c zC>PX{)O%$lb{0+9Vl2N7kzDakc8STfx# zh+=c5SVxGbRUT@IIoF4DskKw2>#Csz%9Ib!Itsc1Sh}$XL>nx~yeqBoC3E-cLzW!t zJn*jjQtdG!6!z)<-?$A3=PKacy&3xopiy+CE|%O70)DbTVsCc4D0O7~`+cpjYwBJW zV>SYDxoz-*$QcI+BsEURx?;k{#Xq=d#vUQ&`I$eXPHM)gnZSIpKh_iB^as)|CC)4n zG|JiwZ?*|>a_`rqqtow`g}D@{lr^8C`9D=4)$l_&XOvNpx9d#|l1JlQfo35VsGzr|zxKALtq?-n9JDK3pHTHcXTxIrX-AKU;|H7Y{PzU0Wogthn z$4+=V;yS({FX!@%u8l|Ptqbl7?&*|XhCJw(a1-o8((!W7PGFvnTe|o$`oA^kIkA-J zT8(Xmny>io8z0Y$cK6{f6Y8@=b_1Yy`R2@}qTnDJG08x4?hfhV*Ssx=g#_hn{F0d> zj`?K~F3p+<3BcYeWu=^Eudmyyc1`ydE7X&BM$!5G)Mv&+$Rlscf62Z)s zlUIiJCPJJFW~l5uxqwwEAY5)}$%-yP{tqcY4~YJ!jMRa{P}$fFwGqiX$K>*aU0U8b zp06*?guSC)!i=i~(S`_kwfHI%bvZt&wEK%r>*5j-&X`7%W&N_%A}!|oq^JC04G6Au z;eT}ng>zQ*X=&%Ro*)g_E*7oNGQonv%Gb#k_4lN@-$)ekD(ufLG}AmS0%7RTzcppoib7Fe$T3>WS;>CV1`>SK!?cZu-JR zFrE zboV31HUVIIFIy=ET}CY9vHe4NFDNrZ%WN|?8Y&R2JxoQj4HpPIIIO^8DIu*)@V68j z37{T09*%7lrjz39knP(6nZhW5hB!@Bu}+{7SCX5hirdORxH z&ClvUGtdMZO#>q%wA3gdwiDHw2=PT42MX{n8j8u&=F2697>el$)5kue#4}gg`bXR4 z5bKE&q6a1a(N79;fpif#)|?JajioKxS>fD7)6x>6-hPfySBlF*O{A(N_Xlx_Mb~1U-NEq2D`R zWbVS+;4>rAwVKiVHMp}z`OA6x#R5oT7(X+=A?PY)`wdlTrGr3Qqd%QjurvF23E~&7 zHmDP(IO$+wnyKtNlC$2^OK7SdXkeO75KKg`6Cdld>%%PA5nqXRB?iztFLFrnSw10R zo09IUjP}Q|qCJ_XEr{MJu{F~>`r>LL_cf-)!G{(p<4ig(7n!>n7fa)t!~8iJh~mQM za7cPtC%aqrf1)Vyxbc%>_Z%KV1pMt0;EciaD&|cha%YvR&KjEeCyoR&Lu4SMo$_f% z0eLXs2UfR{Il5}@BIDKV~f8%F`>U=!oNeX1_j=d`eWDQ zpz=zMkja~AaJ?#!vT~+`1Iy!!7z0ub)?`HYjX+q^%ZV`8=wqQ1pmSH|wmnMm#f6ZK z%d)d5SC||6hna7!K#AoO@e?0$g(|#yh zFf9J*>83h+M+ds%GEq5m?zpPuJ{H7al;ty582>BMS#u*)``XXW0C8p0^Ca?PHttL? z`k7w3k4a~l&@oqBE+Y4*;pxLY^|<|PkYZ-hlb0m=```H=REg8$I}v$4sCC9>=a}q5 zp~4z9Akr!Fz^F9#8=qUmAH=oJwj#}9e3fw!vA+OHM{gdC7e%-^(evOaIDbr{_>wb< zU)$=;zp;q`7dA<*$z1+;fiA&xj6vD$E_+__AC^KmUAkL<89M}{qr>(7 zRG^tDU2&hRVHdan%++u3gvVr&&JF2FGj6vKCr(RqIfXY-F9f2IQN=MJe|Zs>6?S#l;`Ije|m|a!CR3P8jw+*9kD+no<>fDVpCg=&r_Y zmzuyFczWz3!UdL(HN=*yg?Kg?ZgBR2H7Oc_wJs~-NvSe08b1@_ipdYGZoV}xFCyt& z71N(Hg-}%=GF>CeRZaaf{wu&$)iGmjly&R)E5MvC(+8J+c%|XWBATBhqZVrmQ`grf zyB3>?^az$IG1Evo8QEK~6XOCw>R>ktvU(@`k&NUR>o2 z`AcgWN-km8V{+p~KeXhS#AzL=A9qcG%GzW|0x<4`gt`)3F4YixMu-clr$rO}YmZHc#}v zGuu*m_j*>Sr&VNiHdVacAzmfi#YPFz$t4X2UIHoh2QYn{zm*Vp*6bfO>V+I8hjhJ48AK zPM&nt@?#-nhn@k81QC+J)YJduFvdV}>P>`Q%;AA5t6($E66Bn!H65-$CcqqQvJakc zvq?M&sp$_E>3)nY?ofQ5KeJv>V8B>K) z8>xcgz)WC{+Z$u=;7jhaIF{j##nwWohjg56cltm4DOO@Ygp(#qs(ab;29@_k*<^q2KbV;mqiKT=%wa$JXT;nm*?FE?~ z3^_Xq)R?%k{8`56bd8x11S9Su#*2#MTqV8tVkyAZ=x#uFI2IRxsRA5}8MTQ*p3}+` z1iB=g(dZgT%(w^0P25#s?Yu!WESBTxSmfR!5lr)SD}X=h0$p};W=7Al_^@y%E=Qu8 zX>S(fUf;o`=*P<#5-G*@uSoRe{%Hj`VOiHn7ryQNK#Y8OOT?p( zs5}hdA{vGj9sqG0YtCWH)l9UZP`4pYx<`@|h2u+%O@Oh>g=x@8nYG%3#ZUNO$F>`z z|#eb7Mds^)Mit&^0H-I|{m0^l2E^LFq8)bK$vTaAB_&-wJ3g z%IuDF>&^BEFh#(8Ywh@n2-?j`l-O9L(~v-o;hg}+7T#y#r;ieL!FYSB$IW_t3DRWf zFJ;<1^qBwZ=-{H1!f+5TD|{P`D<6j@d%AO5ypsvN9~d8rFmD;FY?y{(g(rYq9(KbR z$nGe>%pPtajc^)8@2d%Qx6o}_M6jbu1o=SorK$}gtXE| z+uVsY80xtvLEV*5;|-*9(!+%LU-~f@ui{%Fq|Vb7>9FJBr-~RDk}93mo(6I4da#{j zG{po#t{tyrA@84A#=%1@?ue@~_8H>9t`VKP?Ipr#;2o7V#!>g*fyfvSa}=)@&xt0! ztXg8mvk)%3TZ}<{)^lLF-t1$@;sYW4(xE}ncpk!?M*3(pT`Y6VCxUaoXNgpW?h#~m zHSJ|Z=;!7C$uTFRLDSvWg3bLdxgK2lMQDaKWkB*)?4#A`dDwyG-EBfGfWY=o;uiF>eA_ z5z2p5vL4%sV0`2YG7sc$JYL(3C0^m1d6LfBffjoe$SsPYFVes+0!WN=6@0CU$h^fc zvomQ;aa<(Wb!D1qpeNvvT}<2kH9oYM#(~!-qL~DGkd0q2J`HMAT`1ZaORL(9Zv>Dr z%8N>jdBd?W%H~3Rd-RDmzbkltWh%N=h*`nls7=Q^I~tGO-vo7mrNWj)itI1;7wq3h zv_Rs;@huRhiSd)xihXlfK6LvOcCt8p)HJLsMYsI})P=w0p_G+a<+xB;M+iM zFq~IQ*+?z#5J;tDnG(;4aOX}-oI27f?-UqPxgW5u3d@70GJrQJ&Nu|+nGEqpXG_(% zRG_)dOw}9nz667nfuq5A zOktM-p?#|6QJgY-0B8mz6&}nTBgl&uq6}WYn384P`I2z)fk$Oywhtl5m>!Ii%b^$% z?cSrq5ScRb%M~AiI!g%&DA$=!migGfQpPr7#do49BUkEp$bqnBlg|E1TrZMOpri48qi&U4C4?*y z`4A)GYLV_CNblTC`8PC0=;ZKYhQh6HteqQt4sGUf^wu2Klr6}aCWDOhM&)?|Nuy_U zIG+6vVK-`5a#{140BXUpRd0;@0)n}g$()g16ID#`!#3KiMqM!3a(Ab~$&@srpMD9H zy9!-{%a~sQkfm(*UQ*Z}HHY`aTLR1kM!syCzAO=0e=$;qzb20B)`LnEcQUcvH^pxq zTz{ndT~H!y(Xsed+{Wz*Im(mPX3 zf4p8fYk?tR>dx9|d{$yfQKnYiYc}Gwx~#6p3?e5mrQaspxx*Ez@TdUS9QA-?^i{tH zl6p0by^e_7JCsZ{ZW7}9BtwSVMZ0}{Rv=#8C(zmSrHg06!8eAcZ@|^& z1#~Vn2NNLJX2M(+c^5^9I8jJz)#YkjazDn*^dn%mhK{?G`>lc;Go2=6J6z zmtk?E2&d1UUXO{8{%i{Y=8if%eY)i4$$~uT^QvL&DAI#j#bTq((K>}+eqnG*hDBX$ zps-nj6uQH%DL66~KX(#I zZc}g?bcMn$GZ(ker&Qu*K^~rM+&=5_^8JcX*BFIU9s|E$43LD=MO=>K7KfxD5_)0- z5vmg#sIL5h<(6j`ML2z&Z_}E|+B61BX8EHmVRfl$yE5{CRF_BP$XHo%^*$3MT#2_uz%1knZM!yE2GR`0 zCkYq5Wm&YJK;rnGZ%h;El3-*tF}}BK@edbsb^d^H_x%Fh;tX$mq9C&*%pJ|{H;|%a zu>IFE0gMu}#o>zrsUJ26z439G8b^1`4s@6w{e%d4w9$l|T^=jMEamAb_O&wsJW_#? zzzO9)(y$|q{8@;&&N98pa)e!ZHX2e#(@pmjsD?%K$FHR*btZOd@v4YC1)vQ>EQ+~T z0D?CI?DpYZEFju!z`2JGhnlgVFc%!(IdTN_i~JY;s-Y_{b^Baj(0`4lOW_{M)0|SJl6Vgsm%yk|EEP_^zbQhzUVwEh7onrS>uB_ z5;7r9yN2Q^(YdK5!E5RKOqkPVI>r!(ZSWdJH15CZl1nX?6Gma>>||77YQ?L}U2%^H z_{si(J?{TRd3j*%veJkh>vw`_16|8y9$9xyNax8;f>j7XN05uAck3CVow37MFxM$5 z7~C`^9Z_A+|c7wPRu6lo%6r zatjm>3M17nrpzZ5cDs-&z>M(#?pk=sjekuj(&R!eeUy&9<3uobnJVi(0_Cnka$#^z zE!%(x%SaV8Jsh8`Gl@vBxM!F@W{7Yan8nd>alQ~r)yAQ5buGf~Ryd3qh`(leBhD)!4z$?|55;{Yq{n4-fIW6fkso)H24Z*7WR#pN3hIHlu%xBLj1|`*x+~KLL$GL=D8$W-(PeKxhncl_`v0gp>o{Ah>i2;%1D z0S6R@Se`xsNAPI*p%Xf(juHCH!70h+?%9Kv6L`R6FvvC ztL1Sap=M`0o=~(yUa<_A7g-5cHxx=K24$|RZNq zu^5&++$ZwC1GTaE?pFLu70~dl#9<;F6g79rSiTnGYizE~v@5iT5{s{``6riywgaU^TpUb}M?{!0SqhU( z>?Usyoa-ccnXC>#%;Vt-VzlVr0l?*B`!y8*E9tgr)27CWJI*A5|j7h6Mk^%q*bx8HzWZ9Mv>?x+ofyl`66xYm9c)=HIRGGc3%#!MuSwiijsrmz)1=_ml6d!Vrf z1*o$i-s_1CszsVo;$b5OM5X-i`Xhs6YqwTi`@qaXOd_jh76mozEx{ zj-hJ9-$FsweBO8=x_w-KiaQh8PY+-1h>h<8PJe01X;eV!&HsSaU^!!a3N z^I(!Yk@lr65iS$X-MG7IVTa%tIvG-HdgAhZAl{V7*;`Mm3-Lc}JXttBn2f3oSMGnRTq8*#@U^lvt?VJXz?RqA{zB?h!}88nPY}MVQ?Txg^#7 zg&=3yZJX4sXE+9L=nlrtBHWj(?;%y6-Z&zM;fU84JB$*BCfDPX6nTG91pNaGhwF%+_Yz7&rvN8T&mO z*2r1H9D#wt!x*B+cfM%h;MzkAVk;&dS_GV~dj46otIqWSVz(JF|6xGmb36lz$Q1If z5|uYY>CQ1H&3iausfaz3o+OH5R%9k{nF!a4DNSx;7CfRr@aCgC(nW;0*l3FI#JXtD zV)PiEKb5F-zh3~lJTixt0YAIFgqjPSTxd0xgL4(q4a-(vn*sq&zg=#Z);zMHIX%|m zfRY}qU=z|I+T7qBs-)$4rXbQxud1l+&J&<2C+AkX99_gf)|qVTjun)f6P<5Xw&R4j z&!QE=t^wcYbh{e`yL7DX)wo|YDa)xvBaS`>$`$l;^RVKUnP9FU4%C^X7d#dKj`KEO zGZqu!-bUfk4#`@d>j^hknHN}|b;|wLaeywRcA5;6-AkldOFOSMj(6$|l1dydBJ-xZ z#uH5gvG556l-x6s?3f@5G*U-jWcw2#@}fiwStI8Ffz%rJfLcW2IMFT$hr2P-IJu0L z9>;`Ncw!Z3cF1)J@Bg6AT1Lf}m6rXdgpfEdTS`wLKFKsQGHX#^TzV4!Q@ms$|NU7I zu2HA-QJC-E6J!R%KBkgYPA&j$Vps&vD*?8Un9={>mxOR7Q7DwMDV?MxkkYaP1LKE@ zZ>v+l$QwVhOgWk0xKxR6{)&HcLFn`HSh0d2`hui67$}wjIO*d`uNij;A~n{y(fCkA zG1KCEkX|<$^PCFml5uIp<0V?46X+VEx{a)AS%q@ux)@dIH!qLY#%|j^x4L6=-Hi^N!u?Iyu<`P zF!=$IMRuG`w!Wx{!K;@N^xrLn)w!*0TAcn{$7Vl(!bMyl(mhFcYs{8|al0@#1)Yt4 z-xm*-5W&#atj04Xq}#kY&m_%#aS@Sjsi)E7dxBg#nwhzLdjaO2yl=!$N*ar_O57#d z1yVPr(FbGn5-=JQA3^A8i!e7k8NB%#a_P%J%_eq$J=E-HLa21DITlao8* zkuFpR>;FKka(NM3qHQ|EjzU}vPBT)c<6tJt(NGdbuD{k5MIc?sq-Vo>g^?8#t31#T zWs-9ocZG3g8Ka$v8JScwZYpC8VnV&#l|=%)Vi#|gBo5N!yI1iq?30SN^k)tf?Q-&@ zWmh`HwuE`yNP3Q14O@UAfv}@KHWugV8)j){@2InZiyMWwRTv~#4VFnTne*a?>fd>F z8Ay}GyP`?59Z#9P*YNE;S(K@oCaxz1(r(FN>(e5<*5j{)jSt5i*&Ya%44@frmuckt z0~+Ezfw`@c2fc~E10YwqCjVI9WH5&NsMz;f2zLRQO|b6&Nr>aJIif)SEdjI_=n3)r zA_NN#hCR;+$lZ_myQfAUp5r>O+~YWKBCZlQ))MNF@K|NvQz5x=}pt4=zkp&YxdfXce{{e!u>%801t8Wi==`3lDTk!uJmQ7w?EGfbq9AWlt z#Vi4iAg>5=d+@X%vq#cywnW;~_Wcpm6&~ZR;2M#Rmv409=05>t26B+^ljZf|Hx{v` zb0fa1$kc&w&~|$M4`pp#9A1#gYE4{;4@Ks|IU;8^^ZmtfF($-8vj}J3ne7*F8#B&U zT8dq}b_Uy*;SgDJMjDNu3i8n4y1yB>mO=0kgTCcxeE(*Gz)^OG@w~p>Fgj$>$M(7f zgi5e`9gHP!g>a?wyXCcbP?)!;uvku&?UVEw;ZBPC5)Mgm&21oLglI1gxSelTiHo$l zoE8oXa!b@uK*vl~TelVN0L^1tE9e`dJrEI4w&Ujs8AdA>xD(hVYoRrOF3o5xD=fEm zUt?I^V?#lyt~_e0Z}S{E67C&ZWq2;jyL8s=zsupd_ges_7g$E5<@c2pCFQi$Mq>=;zI^l zXNGzAW+2)H&@f|ci~p=|YR=2+#Jy9L`xhO!&Mw%8^0tS;DT$0G;twLmk79(9Iv73R zA;GQ?vMC-enf8byG3dHt2NC4SYl1Q7qkNm0NrbL;TuK-@bVx+Ig9tZ3y~$pOg}qCV zQ^zktuZ#=;+#LpL%xXF}Rh^vKAo*o2$jmQ0sSrirm zd{Yh?IL4e$K)9eB>$$NbsS#nGtvdRL;&hRD5bK?V(YRfRY9d9o4@JP6jF4&pOD&fb*!d#lGLRsh*C%djNnof z4OYWY`xbpu0_{qqse{x*i1{Ut{4H&=;%6@ak|yVMJW8Z&t`WHNgulOU&zK5RTq06l3Rmogi$h5PhDaPiN7KiPwg2K)R4^)7D)t-V^O| zNneNiNa=TfA=CxovX5g8d!vN8AXxMJ|jM&8cnYX3i z0aNnYwf9DkNQy(HXpf(ZGT+c)W`+3bUH^@@IrPovd(Uyv|B}A&X(Dqk8fs5oeJJq# zLUW(Azh*kz>jHB<8N&>y_*9Tt123e0-+Uh=v$Q)ef6ohZMCJp8H;q{R10XjSN3@tG z(q+bxJ6rQz1UO;buILTRgN4A8ejW?BvJ8$tF1p6i|KLwg{tPS}^-?~WLp&+mwUb1V z`BNk3ZK0lN*`sU3rs;9ghecr4Jz4l^P5P5iO3Jvb)7bBb9Pe1`O^L<+X*%Z_TZpEZ zC%hkyn?WG5uo`M&!6$_k07$VZrqkhrMuu`aOZ&UhgLi#nn<{+AB;IZfiNqP zveP^9VkF2MC*9VdtVGu-F(&Htj$m9T%;|CqGZv4OVWe5d=BgEc6GmP&Y1SIJdqA%l^EdL%T=u5D1zI+CFb3&_&M| zPrb2|Fei?(M>pos#|d##9AiV&fLlA)?itYq{wcq8LNnJH)4cwwLYl5~0jg9SB zf^x~^`9x>2Z~Yfcb!OtI#6F@NDc!xnu<0elom8^sJwi|(5;S`%XvU@E3G?WV!Tz-9Buky?r-wH zYE37I14Wqm-5lt(eL75#ONmL>K-w)lB+NCJx_m9E-mE!q5m!2j+W2fBh?bS*OINgu za;$W@m^9y7WvoFxS?GPuq}xicc8m}giMLU7 zsW=*E3v&~c?~o&nPPlo9(k!l`DzW$i1f?^zP`gT(TWbm+dwJZ7r$sm+u8B}OWtXRo z={nyAr>``)yS-lDt}>UV=&&W-gk=_Th*55X*V1?98u$Kbid#n**)X7|#JM6|F5Ism z5~{{;O9<^_EHy@B!orR+UA9`AWr9LgVn-3qK*F8$l;rLu#@AY<8upwO&K#Yb{Pc7phG#NI)%R)F8 z&XU|M$6-Ru?deF5yW?a5ZuPEef4nY|9LRv(!E#0PiBporzD)9`!LyIA|Q~mwQHM(fz4`= z?uSfujksa;A`VJmy07G9aHl{AYV^_T_WL1p?wL5GN*Y%e3UinqdSTom+MKeX-YBmj z0BH=aHzDR+lW+3`6=P04Drr&C; zR3ehyqA#_(5L!=*Ok62~X-%*Ca2qAc>>XEFrPDrBCV^`QrkHp^h&4M9lH_}Sb@pY~ z;g5W#;R$BWoh+8Try|%j!H!~Td@34_BYV(cf6WPyZk1lt)8VHfYeGb5Z`Ze zEWDK}ujwLTM?YiwMSW8zt;I1k5yHvo1tFdvgbzmxBm~mR{^*?q;bP)4gZ9L3niSWX z3``54xMqJ_hYKK!3QNn0`p$(!h=V>la*pQ(k}9XP{@8sAglmblPAw)*g?67{t1L5R zEJ20253q$#8Wv6)Kz*j%)`hXlG<>j-c>n5c6YZBdKk1&`-| zmIbH!Qv|_oPW?Q`9={byp*ghn#C0N^EJs|r#}@+3ch*{|lwZ28!=ozP8^h~CkQV!p zUc}6gi_D?%-A(>q3o)P2cIZm#MzO{EfG(8gJ$bh{xrDS6<59afTae3whj9G$Us;Ah zM9GCL4i7gVOdjj_qexmRs|#}>m}NQH$Av=N6k|1p(nT9Wn^!HiR%|>pDa6Zy$9W|c z-fFxl$T2y#Ap-ajw0apfw8!b2!NLZpif-aYP%f<`P&mmIkQbIN2AhxhhY2-5kcCTU z3F*k;0y95Y{k287NDvwE{8HSYZXvUPwy7v@T$;we+Z?`;g?rl2FmWRWfp;`Q!U z{3`#S6$9TY|Ho$+<7wqhiWGH$LFIoeL}eI5rpA;_`A>r!EkQ&U{}AaW(A3L?NitOaw_tNZTX&88xim=>#r5e* zceCEpmn%Y?V^gy94$id+i_nha$~ccK19=A$okm>tLy6LmsUX)LX>%X?KJ`H|@9t zTpOq(1-n`;wkBy7E+I4$BaSV~TS~H=JS7knI z#ZC(5mZ<-zk#8svf};raKpD3`-vql>5eyg}Sv+nv%%s^D(>E|E)Lyexq1`qLw#*@pW-xLh!gyF=?8Y%bWzvxfG?_4`7*Mu@l(LyyKj zKQ_ep(M+J@^PnKj4eYgPox??XYC+~MW!k}5?k8XrMQ8VBtRc$P@2m~U>S!||Ztfm> z>pyaEtxLl(aXv*94_3r4icSJ3iKvcX)pAK{xN5{ zC6dxNz2{Sb&IC!7#0Od}7e4^h{6(#fRBtH(ULZl)?Rf#z42b}n@>fMssOW1{P{1117alO8Y#nOSMPIufd+Wg_|5H9oLzMq27ct}+HV!jH58LVfM zIF<5RoG>?NjRk{dNg|3P1e56seRgbmWA*JlV(EEU2>Ixw!l9~6%8q+kk=N=XgvXx@o@kKRPLOL2FcQy-hSPFT zA8(7u4M-PGiUMnNgOLlvhADqglta+ln*DY5o`(o?$uJohCf_S^9C=tBjQd5o$yntX zE6ddZ=v9*^PKa6h_UVCQ=R<-9)cyLyu^41Xt{)cQk|WF+igTw!Q!plL)|X2}QIDi_ zyG)P&s-@RrH)#|RxopJKh-f#o#;NB2u2`%W&=te;Es_$dv8*u4$pfabeEzO7Ra~U+ zJOQvvL5YRH4+!)WRE4jbH6dNvbT#^{XfvsXsT*t=>H~3(x*%c~R*fTsxxwH!qM5On zIa@GvC9t*SSyW-U04N35lcmauv&H2)LyDYN3eqq_$tE5YVTN)w&rsQ+9{>gp;%K-d z{ws>GoG0jm0}#$Yx4kuOUt+lypobPWQMy+~4HP-%3BsLl4-QWH;z9u=)gc}5?+iki zFE#X?*AdBo%@($VTZnK>8ip?YW&RiW-^93LsQ9<5-i`~F7%l)_{ABVxT!4$oNkXD_ zczA|DTCEq~95$7=xKV&PrCY}a?iTR>{4+r3 zt0!EgKHjbla>>sNx($agvD(4VPQR<( zm0o^YM^LTbk#CRXwJOSnYMzspAwj70*=zDMVlqo*DgsjNsi7ISv#b3 z0r#vx`VEhd48;2)T-gpx3%FuF0>BJGWlb6qF;qfaH_1s-943fv*xep~(kIQqyoCfi z_WebXU}Fe#v$*%j0_aSV%b}wH+%x5Ejn!z2qn!l%njS{Pp&}_D(r7o6xjA)PHe&)7)MAFx=wqnsZK$O=j{GVWB%1VMn`Kg%>G*Fq&y784L zbDFy=32@=yqsIc89aJ#QG4Ba7^AR8F{VwdYj|0;f#ffAbC&F9}h;;^li2xXRX9$qA5~Uw<;2uzm@}}OLOQAW)Z;L zNUDY^(u=~1K9mI7yPph1hUp#AqeReq)|nE={*q7USDC@Ib(3hyiBTwWhIm#a(Rc&6 zJAQBqgt^f|RgKKQ62jkHeX;rc6@jg_FSCci!Ecf{%}INw4v>PCPY({ z){8h>RAz$S;9K%EXs0Yc9M#xHK<>hRG-tmuNPm3>=j_V`P&C>R#dQ&8NH?b|9!QES zP6u*>tTbst+$z#pB5_F~&DdKs@l#^cMR=-dO5Vs-D?RcAp*fX-O1gi%QIIRz+rg7*jhOo^5DF$uGwr}* z*$gmv(N*Rk`@R5IWu-Qz*z11{?H)3I42ypYgWr0~u@)Pg1LfhV-Hvu-QmZ&fFhy=L z@5Pu1`b5fab3efDg9&6AA#Q0NSU?oaY>UjM8fWPbk7?B3s~W%fW}Pj*90~Q# zjv#>@n!#OG83y61&Pn$OaVOwWiOK87O9CifUD`qmo?HCOxtQ(ATIT^UsD^l&c4K`L zwKd9aTN)e}3m_`5KGfo&{I3|uW5Sh~c0T{+*&WGUy6)IX7!1>uRB!xQl*`hdOk)2b zz${@gm*vZb7Zi!0tiuKp+3BvrC=+s)Xy{urcvp;-LC4?PjvItILCyhHo}5|vLJ&G^ z)~nlJqysQ9%ILl+#Jon($+=il!C;QX4U{fN@5yAUYb7@REm5cyjm38Q7*Vb_x)D7C zcqGFlrO1X?L?l&zt_;PMxaMiR&s^P>YDk!I{FfSUGU0NkT*Co6Fb{Heyl!rVtY z^iJ*ZqFtLo6o`=vTv#G{Lz8{SMM9hjwWHVVc?kdJxF@FSH|2(53R*Bf<2FP zc67utR}{J6K}9NH)mTj!m6c7D2E`_27`O zv7OkwUZ8I?je#X)+*ivea60?y-R4Sy(b+Lk>55ZCI8k&{Sx``bjlT)>bVTOK^$9^g z7Mi;tk9m;%r$Wraq$La!V)LuYP+0NOEZYciJFvr>vVI&=0tjVsB>o!#Zv93DUnJkV zx=1%&lc=B}%(=^vIbwpvgt+K>no0|QH7+iLNduJzxmw5)^M8*nu4ZPqs;!a~X|8Md z%lYDvk_R9}!%S&<{ZZd;#}N$2ZWoz*Gx|w`l)C@Gw>uU7DYzx=ik}I}Y^CopQGHn=yhx&K7WpG# zoUi0Oow02R@H_&&wJN}=P))^oW_Bb=#@{R>(GlU`5`Q8R!x7)5Ou-XGIg=jtkGwX2 zBLD>_Y`m`TeB#IN7)oXNUTBw_90)ywI9}gQf&C(){ssQxKPOF_636S?RnK>6aoEj3 zZZavn;{K4m*MkDhQaKQzzl_EU!Wb;Lb7 ziLZ1U6wQwqgq@rS^Q_g)y+u!KDa3h`0)Dv=Je(=aLlMsbXmPQ7<9V^$b3%Ijc?}V6 zacmm7;SlYF8x?uoh+ZMCfK=l2#=@*J63?nN;w#Z`TOJw)?wBosWMMZ`2w5MNj#`q@!`FWC?)4O>plplt&8DWAJ1Ra;x3_P2seNz z2h`$KK_2g2DERa};6$o2P+#;xh`a=JBdtIli~%T1uyAvF1RuHDeU}Jsxn$5WQwY^a zeR}SPAaYOY)$<&?tIt%GkrS^#YGn4~@x;6kEDkW{%gc&rFh zYYwB*vttQpN<|?|02!iLI}ocr4&ef3;y!kkNgHDG=Lqcrwve}SCAjHd3j{wR z^6FX<;%08qe_HXm0JERX5#sE0;(p|LP@0WBDyG&iI1XB8%+U1aI|a}dO*C%WUgSGx z%38y^{-}^VtJF9iGB4fy5|9~DVaJ6;pUGmwmjTJ6V~G31_eA88R~gky9LEikYLydPz=F-Wc&HG^_E{%9V zfcS`PcAhu*&J18K$2KE26Xf{qNIs%f2B9f9z2bONkQs|ziDpU4w%VHoNKHBX>C*HN zfp8nGy{goE~UtzE+@_%hnEQB{ST;!rTGrP<>p@rJie9p=GlfwvG(!jD`7!`km#CX^H}f}PPDV0jM*bHvv9E|6RP*aV z+>rm~u5~nC6hV=b%dUZVIseyzvT!W*4*!)hjHJ?>_L9r$k35eLVf4yeD$S+U1iLRF zszV^XPMHAL&fbhBt%V(C3pvK2Cw3R$QtDkEZfu_~gUGicj)Th}vJ_*TcvA>vmoFAB z2Y>voNi{Ni;Zc!htegQS@$Cn~$|+PjCcFWaLG5Jf(zMmc+W0*}x{8JW7ZM#2?0zAR z>4BTm1-aJKlg8iK0-OoH8JJttfBzwnCS@|h&yol=79Mood7p2F7uSrfp;+SsC@02^ z6RGbgz+I}v3o|E(bd#_ap}vDWJ3U6V`#<=DBq!4EUeUJ;KoKQoV+Q!ahk%|yq_b%w z={QO#(Z&#p-=c3yqP3zDYy1<60<+f`jBP%G$fH)y5?Z3o9dS*k05#FDV7DU9x}>dk z;m07Z4-O1if&VE$<($wK`+veWS@g=Y&_^QjNachp4T_lO(;{|vWrSO%?+I`qTo%gZ z{b+d(8u{qbi}PPW#Z z-p8|*p9wJ&kZSbB(Vs)RFkZ1p<5Xd0eOiNYg{>-9$THZ?+6u;NL(_8}S`Y+?fG&8(d5c#IXV#360h? zdw(LxMdrpwr}Yj03qtMsaC5zzzVnuo<0RSy+G#N2g`2@-gn*e=TljxqZ~~3u?pR#} zgTvm>V!m%|J)VnwuEe53Xwr5bpy-LIqMROEiE6S#*jN}z;Lby3zf6@UZR2CFy4+S>ch!pUzpt<#ar~FCG&BH!36b zxNa`KT?AD*Sw-`>o=`Uz{Usk8ZV+g;PRD^RJD*$2U|LQQp3yz_nWsoG?E~#kswUK( zM&735LeVaA*AVY(2)8R_6nS<6IJ>&2o&W zJQLN?^VFOf@0K7MTpVR|n0Qa1JB?nRV9ORC3(B2FN}Pl7A0aMsXCJbgwdRL*nbZaP z;`aim15<;#=6(x8=1mhTQmd7Anx6=zh+VuMfyc~+OyPfo>2@Je&NDf-uEsBfVCDI(xyYo_-8jSx5bUBl{ zJX0!zaq4$L-Cen4#?1jW`ca9o%ppMLOuDTw7eeldC73%`kgJgdgne;I8L=jhW*UQ2 zzDFpk(vB`*thz{%o1_D2(K<#L0m!EjSBWsQc^7~a{wMigy$T*%Eb5?`TJcPSh+HDP zCkXmLP#(+pE=*(Z+X3ckrzJ^hUaikw`gN3D*pfF#T z0i-9AE*sbSJ^|bmFa{T1q8VlVxG&m&F}}z9n?_1kd@jftbAZ(}3HZas0iCfn?A*3C z&qb3c{;jwj{mxPjsbzYqzCEOSF~HbQWUdP)xUhl! z_|jl*ReAZW#jgcWDDJ@VRk92O+4i!-++W|W4fk2n2s%=T3&LG3t_ejutt4KJ`IjxC zbg~m-d@U%%wd9P;%fF+sPzm8^tW&$>50)d83!2QE)exTvb7ovpVa~Wbbl!qU-8pN| zKPfnOlyOz{l~!<6dQS2qb(LtZH>@I9BxFOUKMMEeicM&m`+ahJSbw9$N}t)mG)%XY+s20dCs*$KQqjY0zK3?bn_fG;wUU^RxW~~@Sw9i z%>T(MfG#ZOD0J7@B5${9foZ?Tq+N^ch0!-ql#G4ycN=!5UHYbg-F0rCju&Chb;zuf z>n<4lQVe2z81oDV|hVI?t5 zkZZ(|Uk$fW83^H8QfWO=5b0u?7C&2)Zxswjd~ux!$3{XWfmPpHAnpfrea75XC7|R6 zW5czJ5QsnsF-w3MM5FNg&}eKY&;t+VquMT7?T*lD`UJjdOk@lkHQyHHs_0sv5(~E#QRN#u zHW8f*KFp0yGWcJ7BB1%sdCYcTs|X`q&hUuX4wwW*j`9cjq`uuEoB(iM){1up5jeGT z%$y9B$5jKp>6m{?0dTlx4NZVck?xk5Yq{c8s9dxetVT(NNeN<{Nt=TM*CwCk?x00`WRb= z4TP9)t#rEHN&sBfJ0jPWfA+97tHdV)NRAV)T%>$a{*Oiq-D>U)iuCpL71P2oLd+B? z!_&8@(pLg=6|p;r$2Wx5Sm{F0x95)_T;E|1Ns^6oFdr65YcoO9BKK|N2<+YKan{BV zt^&&jE&Uq-uBFBrJ#fM%AWoA*UU$8{F7wqjHU)Lru;N7Gq`hkbDO);vF!A;&(0Bvl z*GF5B6NJGA)*sm>+_4#?^JALqWpSIdIS3WvspeXYh;aD^&+K-cezGT33)0pnvuxd%JkG42M1)nqMtL<}(X_dwi}SF6eN5 zKw`T&wgh28>l}$m`gGvB97FwBv{zcLY`bwJo8*E^Yz0i+QK-{Ne{YerOJBz*^ZL76 zLpg)w1gkwh6_UrL#CNgVHUO|jn$+<-5oRtEHQkmJ-xoG6Ku))rj3h=)YzywRaSx)o zZ*3vs5|hxM^zG8?;k~$DG!+{`_OP9UO>A2?D%G_G-*tC?gs5Tp!E)? z-Vxw_OOYB{Ll)c}hz9FpJG+Fw^SIJtC7I4`!ra6|%umQEAKn85wy`flHEk~lijGYY z>F>5Tl$(d+t&I6$(McuPo}}A4QC=$0Mbp!c`0NxwLP(3*I=?5%9X&1WwPtdvvFtv; zxuu8NyI^R4{6!eW;QEd@qV}?c4jXj%L)pCdl2SZ5m2S zpZplwwPjDhRwX9?1jJRU%1TeXjwu3N4ix*i#X!Y%YKh6`zmDoLO_+N!CIuXPG@8^^ zV3q!I&Rq8O#Lq>$a9Qmz4lh$o8dGBCRl>|SW>Kc*dG;&f;ieXCXY?ZlIcbJRZ>)Df z5d?8LC&*@eR6_C>!84w*!GVMtZ+w-L32^35!N?7Pp#%-BI8gw-Wqq6;oThJ=7)Bvd zZpFER%s%WEbO-#X5E=xX77fcqDvsaJ14sYY&UfyeFq_B91voZ#EnTQ+eImrnr+FKr zBl5Msc?Td3&Z!%-gsKU~a9dmL5 zA!uRqd%+;^Wu#WV^9w2X*(j{SO!?eEsp^Ea96F(P0@(6~R z=xN~v8UQXXUfXqx&9bzVKo>%H4b9k70Nob1tZ|9H%_X#1V~*Y;EM6iwN^dS87Z``8 zN%U}~Fc+B3Ij5y+yeEiGgI7kz(;Cep06S-1+G#{pka^ZUqFSEFM7xQT=1)2c9pV(^A+-@p917*0k+$;53EV0I-3n4Ray2TM|zZUGwRQSdadyeZ51EaX?5V>@laRijp)j~VKzP<7bAXi_uHF2|O z+6S{Vshd72(rI=en&vdqz}5iFMMi+v>eeYPZaoT+jPSXS6lU(Dq1*wHx3Hwo5a3qf zs4p|M<1&Iq4Qsp@bFmOMupJ}uh(76aY>682L?*gq`?Ah4{G$p<;VaG%<)SbSS$gFR z_;G<$K^=uH*sCH*i^X~%Ce4H}2RO>7z3e7};6R=~<5bbEae(dIoCj9lRfGyR&P z%rm5F^;r5iXqO`CtFVvZx?7-G(4QRa{X&3Q!1$9=-b^8+iq;VdQzwXWzSIq=4EgpR z59WEB{h<5?Sxt#Iu}x^Hh+Z)_%VyhGuZMq8Bs2y;eBt-Kmnom52X>BEVrNC#k3pnmy$iO8~r zjoaHoT#&AGmoV?F0vO|-w<{JF;H;%{lH9aU6*j(gDAgGb6Fh!-&tX;aOpJg=7loQ@9^%qWG?ZK!rnEPOHqqfbg|zty*k#;(&(fL|kw zzzv;#pAg~NHL#iE^^;!$xPRD)j$LYOEYQ3jM1yO4k!}jy$PdI%%Q)0X8w`BX+)@T& zbsmZjOPa~772iIEM9lkPHjRv(g?18BP;szKkTL;?YP@ zZUuxe2m<=zbOm$mS~|$bon<_^Jd_{LmjqE6JtL@Y{F76m&45;q-agh@_E>?=PnUH% z4Xkn+5Cej<5u1;LPKR>t)V3}&>8PFoW;U@as>GmZCxR{+YZy0@We|LIU>YUbOc{_4 zM>ReZLTgS$E%!TT@~xRhR=Asr;Ezi0P#l;4!0Qx>a3Uxe1*QAmB+9!paajvPx;b3+ ztQ;!IJes0hRKy6a_`PVCgL8s>q95>U5Opz>9C_!%1eXU~5{htGyvvFi)pJ0c8riXQ zi*_^gB)>ol{|3N?;!uwl7e=e@WqMVGbzw1oFF%FDZe}XUYUSvwKVsot{99ym-2d#!ABS zEm7I00^#*^-s5e?M&}iY$VZEgP4CSHu_=}tuC>o6l)0mq2V$ydm$B8?>2wb-F$R_#nF7c5s|9)M!7_w(ae7Df<}zUSP&?CO zHEtC|6SuT*u62tnwq1iWCjLv(SND9u91$hN(> zh&$9oB^4!kA!IdzpQ%{wlHzZ<5Mb^&K?r~IPBFI;B0Ohy<95AOo57x1@`yDqoh=@R zr)FFw#Pig&DJ#dbB6I5_AwwSw%l6B_%%Hv&Cw5l9Z(af9?kP8It=LF_;}MBTmlFS2 znCs8(kSl9m=P07FJ)^JQSORoEqzreLfS$eNp$q}k3)xOzOuiDr?ZWvOiEV0^p9!UR z=+QM2iMK?%yy&x3YrV~w=PDrAvW|hrS|Z_eXJtnGsQjy!xowfF`HTE`47w82MYx*m zs+rVI5+ng@u8LuipY*o;u1_5*+?b{>x#dFHg86BH0X^@ zg_whA$28KrGP_>uSS;8{KDe7`hv5#8AqJNZ5~fL&tKNY)T$HmSu z5j8&W7@!ojJ7$z1?%vqeXzlo`K=|FoO=~Q9J%qDQ7Z4YSrr9JmXG3*^NVq=<$7xsJ zz_*)*-4>GiJ^lcYd84aA_6c#1Fc(8RH*1mX`^O??+Vw|8fQrs(6|Hqq6sTAIbEQ8) zC`a1+R(+Gf2+x1;UN!n~U2(>ZV9o%q2E2ZyLr((TwzU};AGP9Q1#_z;Q30D-iP^6f zOda&dq~0@)KMQdExf^ba1^*09_HAvGVnpAraA)5ru}>0E^uKfw!Fuq`n?TKm5gdZE zqvcvan6pnCSSei7*n0_firnwA$-k)t@Z-s-{_bB0k*mk#EaQe|oFx!;;kghFy(h{= z<`6O#ABlG5F;|hY%bYiZ@bxBuCaDd}+rovS##@bN zL(*9PKY`9fvUCJqua~J{_Q42xuLLS1mKW=a^1>+gbc zh2+Ujin;%o4Pvh?q1AWpCX_oX3bp8Lq?DEr>UwgA#fcjR&k&Z=t9KHQEeCmHH#I4=2s0oq^8#V_cC4xaq*zbHcYMW~QG z#HUNcbW=^2hqnkb+q%%!;R645A#@i%uNMnF3`yQ7TJ*$$B0Ly4PV+jsXvgM8SbNil zg}CY1Kp>Ml`w;*oN0T50S^QB*YA>T5F5@)oJt)Y{so_kApocH?n8Q&iuK9G*jay1c znX4bYFNnhNKvgA{db~&nBNy(;by_-ApiA5?8R;@lKvQDw<1wrazD;?)Dort zRps1*vd-U}2Ii_f)ECuPix6G7R@KeQs30ek zwt9VWq7d?B&mh&>dqh%KmJDtpMVL{T7YySmJGBrxkl*to07x{uh&Q9ho*0;u^`23jY;l5M-dbetz}cB75{glHx8Kh!fuf z^@zbIpA?B0-4_XTYSSw{^!>{Om%BfX)-#KxHJ&ohu2`{MQu? zeZn@aw6Xe!MOZzrBiqQR3o~mNqZs+8Jy1e3|NGiWG@kuu5qE$|lXmKecZB7R(TC>= zdeWqifHbX7m=s&#CDMI2LJ6=+5b>tWla{|n7Gf(4CEii5la6qdVRgY6c<(PXe2?g)EaCO#U5~{U71n68NXV z#~ypHrZD)?i}3I!ebXV+i+S<5DD$$H-nh>Q%65e&ew>f4jqj?mWj_qQM+y$Z_&B=9xv~}AiEHz_4t)gmlvsWrkw*0RMTv_ z`8-9C_CDJ4e$h;2xcmAr|0S7%KbPA8wyg(EhVWiwN zo-C3O<1Yy^LOv0cCyi{sz$UX5Z+{EieRfc5D;D_kIWj#5Z4fEB#d3nEb_b0*S>NMb zCOzn>&FU#RM77=%r_Nu5z{eRDKy)_Dw1kc~oSaN(d|fWkRY6Ho@07-qg4~wUJx(S@ ze-)OQDns6n%e0ch0h2eT0}B*^xR&Q-Ytf9fE(yOs7U3?9jsW6T3H+-O-v;(lgcO3a z#L5DQgqO3vm?DB|53}M9>KhKDealTi{H~1HsC03~_HYU4J-^Xdb-^NO^cv9h*j|7+ zU2CS3%!5K)Ei6EKVzGswU3E@2eQ3fp+4Pmm*)8SzDf3Tb9g~(=o%nMD5<8tCV zk->hn5^+Tsl=x98b4q6)GQx+pK=)D^Q_zt$nOrV3*STN2@IMNm#FM5@iTm|!?n@CR zIlNk65is}9Dw{0Lfgd_Zb~?Y_@HxlnXRo@WH8%qMX1nLKv z{=|WToGhn0iW;+oxKH*VCp=FiDY9BsRvE}%W{xcaTVJqv~5%u~z4-p7(p_4ma9+#teC+8?*ECVO#+YFOLiOzez5Upl? zQjL2~6itGtxi7xGJOp{QwM~ij^-a^|7oUz1$pY0g5=XDVrxK9-DCS-f%0=K6pE@hA zCe?X`x_6-Jj72?<*9&q<+HsM^ytastaYI@^ZDJL<<@E={Tl&*5J!WK@<%o_o5Lbz& zoDKX~t@8uEDUFUG)i_2JrRG{oB8xRwf^MBk+SYtlRLDH-$@!tlk6hU+mNV9w_#Hr|^6;!cTErM{bC85%aC6eTt zyjmc{jMRYgRaY#!S`m|zCGX8epAeV7%2VTUqG*q?%1~!qCX(vtKoLudTcMk zeUWENeP7xiN|OVHt=w0I2qFgt*!$oZ}(w z&jMUj?%x}6muT7wEm$b&Rvqb$iYVv6SyDPj@vSw1oE{eqtl$Zt-I*<7G5yK+ zM9vA@>Dx84GQ9Qz=LmEqBJt@S!K@LH1HpCmYYk`^TI+r78P#;OCb7qswvb^@% zMO1E%(duHlT1S}o6U@ytncPIT;%9sX z;9RC}GmZ(GM{e_f99m+R2}NSaF?86O&<5frd$RTRrPz}{a1dOf8Q+fF8g>zTT$T;EXMMWz7L(R7m@2Z(T`dAg=0-)2_` za-$64+DtCR@VZ=v!%AAFCWc&qu2CNSifa`n=U1&s(LG(t-%>_pV=UF8xV40ETY)FD zTHG!OE_S1}_2g8(%|#rxjl|zYtIM>t#a`3+CRgj%atyjrfF?p%8c&Gu%!C>ZyAcfo zDYHymmp@=$XN^S!HQ!9qghH5?`aJ==C0RMn3 zji3RIuH7)ls(vAksJS|MZjOTlI`O{hSez!3ZpCG~*W!!I|1s{2w?%puOc#qXJX&Nc zV=_WdTF-Fq1Yxuh?i|MA+O3O7ohW}M=LmNR%T2{bT6>Fk1-ah59nup=Yy+(U!GQ_U z*tSsSYu!G{wHr#Pli^kbL%Di%DWGQnRN-b|A(S)#`UIOp>;mdc0J9-Hq@`=f%M^xI zLdh_uZpSw@Kmt#*i{Ot*D6lWReUT8R4d}$hib9;DE-7k*a^#Qa55ca_NO!#}-WE-Y zy}WxJ?}{+rI~WZw?co17q159e5v~-*;Oty^A8AJ*^BvtQ+6K|s(!!jr{J?XxNtZ^e z3C`K-renJ@YF6Q9-z#jK`*b2k=s_ja&RkyMdhu34l(Ef9E{p60=@v-4c;(iLlZ56L z=pL$Ssk})L73I1WYw%qlJW(YFU2%K~Fex&FEWT@*uBB~i*}aA49?i8P$o@h!15cb7 z6@9xe^ZW^BIXW%P6qp!HPyM?Hb4>2L25QKD-pc`)=x~C#o0H>J@wivtc{b(xq%Xd? zI{>-Ml>x^#`o`u$ayEK*WC3-;p5 z8GT4r&B^fh*TUT=nhcxUM7jp8p?2O9m^LeHzP7iz3w4_{3k-4Fl!SYt*h{B<9!QoB)Kop)+m84R;2^c zhzw#jRY2E>odLEXdUAwImB)qCA`(u<%OW(oDkE6X_JUgQ99crXPoKUICZ0%`xJUVoY}58g37CUU@m&^dMKp&-#=%$+F|wql5a*y^Vc z6cL9iTF5#@@*heGmH3~CoJBWc5eXQh;s8ioCd8H^oc=J+!sWSgJE7(Sesi>6+ewHS z+mFHxdfPdbVRpxHLMFQOM%9%f-09QKw$6*bbg&l%kF$Hzh=_J0rbtQju~rS#3_(H6 zkGHQa%!#qY!NSW+KodE776RN^=t9hmYWNL00iC4|$GDuntgA>!jn3P@bpZDQ%r3Dz z>*_`!408hE)DXEf{=!8#9ds0VsVgoJ;!c3vfVD!jv-jiM;?jxJ0U6ZngS&g9s|U*5 zr7zJ9Tk(H_97_*eVCO>Bo)Jj@>qdrjWP@+_ZHxfo6p?v3;h~Zad#H@t1-k)cQ!fKQ zT64Z$(5%4C@r7G+23E4YSVjLhyo@kAbd^<&RfW2e-F>)gsKr`B%=JNT6)RB_;3}l! zHnFyE0=Y=(BA0{c7@6-Z*jY%f<(KDU524Nil}m{ycI*T3;Kx**=BD%vGW3c2`2*gi z{d=q-%6#ZW>wSHZPQB7E{VUEDwOHvOP&Z|J-8=eput>Ob?H{VOGtDK0#+6&T=vOqg zG`EIvnh1A3mL%IDPNzHgt&h&5KsW1+r%LPT^wvoSas7mcbO4TdQIDU6XPs> z=a~RGLoJ>Y;NFUNgTa_<1~lxzIXLatLSE_ z7E?x{Twv}+FpQ4Q65;VTIU{*qh#4x8pF3b@e@D{|`+I`jfl&ELy7pc@RyvqJ@`R9Ncxs#k=0t|t-SZiT>y%+rzf2R@ zT8EH;i^Rg9mHqc6!ebv?{>39g%sF&=xM7_CPylt?v^dXo7!-WQag^@d#zex_Vd)9a zeYgphuz^@XM4lUl&{$#4Sw)bW8Q};6BR%GoP_wgD(ZjT~?&=EZ!NFx>Y*VHsgPG>( z0zE^B4XN!91(|mf4-M#=w#xIiOZGYp_@vC%O=x!|n!(!%6b?Gj`Uko8xd6G)ez zIAOiGVkX}%Ae!wOajNfH$AY>e$$1jqUXm*91%e%zX)kRx2z<9tmj%THge%q9=ePpG zI0!c(197Gx7mw{7My?2sVx!{$H4<^{8Rwi(MC#{~Smn4v5dZUX4lc$;Wd0Iep08q; z6TvikTZ~S9n~{2cq+-AY!+;NYHm&*gF3pX()*-eD2&McA+s5qMu=wy-ENGaogLKs|j zCi~r!p%xx`;Fb@>!wqsLBZ>YqDf?xo9mS(&zf&ya% z*TJ=}?W;dIV1H|m?1$(r@%{6`zteNW~;ctl||hR!6EYt$nN z@hs7)J*1U$iYOOd_u5I4wdh#|mZYze$Zjocys>%uJ+0B(lmU?rbD*bf_7-NA>xD^> zkrI*Te{5(D72-~SN*7zCEq@JQwlH_Gp+ZnB$i+tMWLoTSc0nUl7>dV4=k7_tQxE<~ znA1hOokvTLJO{w(O1}jgj+peDA|3R}hT>PEVIo>{eQ~LX+>i(}al*wLJ?DaXY_Vra z{slF*7CsLcHcE%J65EM#X)!EqkCy0^Dw>Bs7vW+t72@jm4FS%E(VkvbiEo_`M#FL* zjOFy51)VtX#q>3;j;**&f4Eceq$`*AM8^%u1qI5A!w^aP#o5qtYc~2%LfwwcP8vJ# z7i;K4Ax9e_>`FCmyD)O59pPND*c86AuV-YwLJ`lg1q{1aW222t^8URSC!l zxE0%9Lce>RAn{ zQ-640nRa@~PF%;h5@$n?u8>wM%pVi(tXj=cNn55Xq!F|!q?B`5nVFvAODssDp(Dzm zQU$>l`A8vjk6zUEk)Mk4V&BfdeO9Ea$-2~xqb`A_w^U{z5<5qP|HK3s|AeC5ICyo( zl{Q0a>ZM?$hmNrPf^BwL5rmzf?r70c6y}B=-@?FuZRPifUiRG^ra^y_vzbdvIUc)Ws37=4xTPrV%qO=N{~(yIGg1CM)+B}6+!Z`#qtR3Yx-GzRy093iT4zff0WpizV7+pza_NwlzB&@hg3+4LUmTGAF`pL+PE# z`egy;OHZY{8SC8#?UJL+E1~|@C8FIhOaFZL6UHTIH8{4VsdL>D#NI4jdgCQQpc%?5 zOA<^~2l6V+%g@uJU5dNmx zC1PGdkopWa?OW;F1&}M>Budyb|KHX&IXcSdgXlABKvc^pvVLO$Z^ei(vrUhiq6JXK z8$iKY@t!FmSc36%E2chJ6i#OZ^@BAY0;2CA{i5DmipqmejEtKFxH_`$*E3`Y6xum* zE07!zJp33CWvo^7w)EqoNS!Ob-nis(hn5o$`lsAmZuErz$Buq5Iz%}2Ug?z`E;^4& z#1$~$;}StxS`TwB`Xphf%A^UC;s^S6Wi^T=&!o$=3C-;1a6B3l1vpvsoV0g1NQkS! z18;2ndG}5b1xKZ_8ZU{UP7D}sSl$sumOQu`-+G#FGfDnLxK+Wwj4-0I6Im}-dIrK} z;>v>enV7a)v4Kzz0?topm+bBn$NL`%axo*NM1Zk4w4)wo^=S&YFd<#ZBB&gU_k}pF=5{2;sn0L_4zO#0)&ok9 z2u_w4=BE^@=18um$94!|HCd%yR!MLSd)PDu25GXRUV~^px9u9|4(_q%<0v zd<^aKr_+qef4wj_8yW~K5reP^X8uRGlbcR+vLeP@p8&ZWoVFKtM zMK}XIPx~7255nA8@mlbF);rDi9s4Xz`7|2JwWW1H9i#HEM3K z^~2Dq5?d>vGe~<>b%hziDA#~(aYQ*g^-;RqLqc3q66gIV$@{?$Goeq-v1GX>=al-Do{){1`#%-ty&WPM!%%vk8sFEhusvvW@Row^|FRkfQLjmVQww8FpS2L!kjQR zf@xyBRgl>|9d>U!Hzc)S1WCD~Bc{w#BqP7AtQ({8nlSpd{G7%j--Ix0laIA#EGvX^ zNUDv)f_b6L7f$S4c=XH$aKOPJL4euG!f4CYxJ;nir-uGq7t&_B)t`jA&!DL}*iC;| z&EFTyWZE@^EcIJ_k2ke+5M$2yLGpx`rZGebnhTB)PBjou)Z-`-Br&ONYTT&r+9V`%{H7+qxgb_lOS`qmnAug!k-uBXoKVs?caum52z1vxp}@QmkY74&cqj$2p4)< zIsQ*bZg{q1I^RE-6OmxOKk5q-z|7G3z@M$NceSRn(KvJxwp+T_zR zF#3*2#pr(tbEE276eos7$UMuiu-S>hJl`Q*vw{?p1Zk()Au^(K{Y}t#GvK~NP4kv8 z*e*FiA20hr&;an%ifp}7gHu%c{BXZBA_m2Z3cn?R0wOa z)S`gqWjE{8_@OSrE|*@O(7G5aD5&$n_@arUlg)%UA7p*1<;k7*z3k~BAgK`5pE^pw(o;@1Qgm^5;i<9Ja?x)J;?U50P&8E zc{3he42o2-CByqhCUoZ=>n&a+NU5_gP*$TYgqug|3cBOWpqWAc98FJrq=4hyrkBS) zlL{Z3`}BtcBVwx$j?ps{Hr{RIKPGjsKPoRq>?msX-AnN8D&rzh`pZ2+T(u4w3VBQk z^dq|OiGK;u4h%8)BL9!8^MI19sQy2Kh>}&VNS3gYac6dC6G#>j5CK7gfb>lFUt7E|gP>#pOB5UfhvB!VuX1uT6`wla_8 z-&xxX$MYimx`KEjC$9)G6Ie2`c3W{>0Qcx#`RA>_p0i}K<#0RQZ?U}~=fV-C%pMy; z+*Y;udB2qw`YU{*_`&lMq3zRB@;my$hGYmf(*R7xXM+L!GX0Xv*v5RW9=})uA<8<3 zEbFXaq+$D{x4ybTf#?f#rf&#=!+jNEl$)>DAJXzY4eDaBSW& zFIL>lZ)q-$%|&?4z-?`@r^q~tq?zP=0mUrc$1y_MFJ3Cxg9Gt(xZz1#<9h_U78(O| z{IuET1=Kx|qSC|}g4{j~A)a7rTNKH(?qkayhYNE3vFFtEIVl7t&zdtmp4PLACC6nn z94WfndRqeL>4$!f%88EhErG5R{fvxW72tAOD4Vh4RzUt5PV)GB$Dj~+OV|&N3qA#b zXeh`aR@b6Pj2Re_-_<$q6EZdWGfy-2(}@>OcRJcV-E@x;w-6VbQ^*EJ*uqD$-w17e3(+B_#6|lU5eRT>!2feHTxG zo^gr)(SqIJcH?DKeNdR+W%!XD_v`}Dg1Q;iIRLcwjE))cA;j6!Em6sj?T#_2rpXsA;@o9a#vvZpH^+fHgQ}wA(nfqol-W{6M;I50maxy55!-%-XLZ66 zmWw-OLZ`+ke1B4b`$0bP7)N5!3a~Z1ZGYsmRpsr+QCX_K1v6o^?vKK zKzVA%fJUPzoRERa75Zh0ZM~c;X)*q+Kv$KJTndVO$gV}6yozljL3#z5LBnlLjjxw! zdeu4DX0G%((wMpI2a^9zah{gqQ^>9$r#8{Z}Au!yblP zf_I0QDj>Q|IC{GvejjdQjq|98+@o-{%~RcbLUX=~?ZaWfa2&Ms3mU zB|-71v`u0l;R`?2FIf@>nzA4xXiWmm&_OJ3{-ZS1nk%;!#AaWBaFJx>tLWATmXHR> z7PtF6EXcg6Yo}C8s-9;`G}&4;Sz+U(u~(6!I-+EwV{d8>R)1aCl3NQyB(_uYZ2wS?sL3+b&_6PH*Fg8wrna_x( z3aAIQ&SWavSBOi+(gs#4M9F~yb1#^zAp!}YatzO{`7xezfg*kr!%)8N8)UuZU&uAv)Yb#1Ub9z@e~nc+k-(|TREcFVow3; zlSDF!?;HX_!8%zrT%qSIy0+b>4ek->*O}j#Umh30>*H83#Zret@Hqre#=lIIANdLh zoWLoXn#Zt6Dx2&UqbbV0vVz$nhVDsgagksOF-*_CQqL9{Mb4CE8cv3Exu59=*R0mV zj*&Bf65;EL+tR3z#x+30l851+2{e~DQqgT0@vb0BWz(as9wIm)li_*G9(MHltwDKlZFtq}9i?Nvq^M%-~ zEmq>mL}-p+^Io^GV4NnvJV5i-kKy)M4MhB9E*Xzqd!bx^9kjt*hUzE#09_^6P1u7< ze5@ZReH`gtON2SD^R|XhpP4?l)(?EXp3GpGJ3lKx!qPb_mLA|)QaqmlB=!>Jo`_a} z7-1#Nbrg+wR5Tw<3fzwmI-7A8T6^f(RbLB_(Lpxne(@*15lZnctPH)XE%`6^}X__>1-4) z;9k9;65|4#NEZdC4Zx+sTs}sMM(jQcZC?6dO?q^#FwfQxep%5D@zZN9iKUB<6_Yll zbHSh}pub2fyvrn$C3Q?CJ3fJ)7IfoiW|(a($}+HQBOkMHlmoKnjVid?i*L zcNU$~XUEBUcK(=X67y^238%zVWOnSFiH6*G-x6V7c8{@W-g*++UzeYRbgQ+vT_ANP zs5O39Je#MZY_Z0et7DBeSZ;U)Ng0bHgitgLAX{4;RYUBeeI*bTm{rJS~vk7 zEYJz1+?R_-9Dg(zg+yUVG&>P|rE~hsxaU}&J+Tb+FNpg^k`AZ*N_^`$2~Xu?Nr#PVemA+A+(ymugW679lr zFe=yR{&?oAU>*>M=!9}oV;7iEOCWkq%)y)Syb_w7gsH6|T`yz=PUx0kM%R~@NKE-FJyrcvW!p{|U=xbwj+zFuUh zbFXyG#;7nCf3X`7q|=|XGRO~>X;z`n!~w;hvFkJ7H#-_WW^>XkH)X{PDYb&itk1bpuXJ6nm~RR|4@@ zR#wNJ36UiVy)!d;+$P8wFu!tadzS#WtM*E$bKVrpB8Hy>FQ)txOml-W24| zVsc2k{O8=_t$JmY(K1#!uRw+ds(`V!5KEHI%%x*y5c!8tMroGp8_tS4q<6;5uf#HgU{trdDUK7wMHv4nJ$PbG=;*ftA!j;67Q@ms! zc$z?e1RX*vz9pKENN3$*0}+=AppJ53z?gwLE`Je-^%JsWW^S?G=sUulzkH(^H^~2H zfi6|@V6QQdyeTa8Hg4i8f5*~^P{_1z<9qyGIOi9BS>*PFelbT`U*PW7AMXfq1*9H` zp6^3jWVDLTCz>1l0F*HSI|`OHv8hN(Q0LO~pXk|*m_m6Iy(1?^zAC-lvBJe9FyFbW zx`h#_0DpqRUMV0nsQyE!Ys#(>v15BQbV-pnH+)HNGQUK+8{v0NAug$C z+KhWlxY7R7A_E3Zo%Nh^8Hkw2)co@w@@$SLqm^_3^Cy9>4@yIYjaltSATBTCxfVJ) zq@63&=~$Lr-bG4?h`iF5-n#AptPg(bw~fs0@@PR}pFz!7vF zZ;3=si8V=;j=3C)`f+FVQ2amyKcT5$)3+#73OroF$BExuY&H@-98E|#re6W-Iyc+g z;G2}hy9o4*r7bkU9h3HNM6gSX!*8-r`ja3UGhHC6$f0jv36Pn?+2pt^H>)x4suGnE z9ZGnWBniV z3@`feritT3@B^V1W9Mslwp6z&oS%PP^b{{@n6jrSb@5L?oj{jeOu* zlDJLjkPPNjW1g_0f9b*^88XnlW7|di;tC=uZ0Y0cLbC8D>*!Yn0vptc2V;>aHOfRhmVY2t3L!3+zRAfj zl;`xdAvgunFMc@Zn#OVv6V34X~U1 zB?u*wr$l^01T5ct^Eh9RW(p&Kj8&4NWV1Vf-Lkk(pd1rThUpYLsa(5j5&;?1}a=gkZ|b%Ao6T*5WNMq`JHiy4;9cahq{qopl+b#@!O3Uoz?8=*1y zuVpg!BdkCbwe;A#i)1>vpfs-uLB>YWV7B6_UuQ~Y|8ch{XDU}dI_6>bfVdM7Xe=5c z=>)yC@i^>PE?k#~{9LAdQKnduU z%7IwvH{{`(SE|W21NnPYpqba?ZiDOqZP;aRjrV95yI2Wx+1VjHBa(^_v&-XJby1dB zsVil~_t)P7xx}2Rcg~Bg?*nj_3W5kbtG@$r>(>W{8d3Xw@e125R9Q%~MB~Zig6%WA1IIJXpN5n)Es@c6OEi_EY> zV+p*S0ChOdG}-ExKhr8GOy3d!BXvp<)1H9vhewFejk^#2o;wNjH`t14i+z9)vr3oy zrg`;HVdT?5M<3%N_-2ZO7FUTfLkI}Z&)1iLJa`7;s3-k0W=I3^?o%c(53R&ie>TB= zJ@{Y#rN~HDm9K8s!IS$=TYsbN!hVZC1vMR2gMm?hFxnKE2#$83RSnI~jURhx05P>+Y=xy+iR z&b?SLCD9RZEEb_Y$Y!}6n>^>A*!B*`4kBC@dK3oQm~1~^K#Uz|J@BLvL+@J$^7PUlT&bR_0V~=2AHiyR_S-sy zBLJ*`+-C9)M=Q=CakkJxc#|>UuLQXc3QCj~ahLrA%yp2JnUB677f5-u!WoLqUxR{2 za>8h!$J$Y}1W>E<`!m7zu`gZ`K`{qdk8b);=YcM)zZR3C_+=c^l~eW1`|PEOhB-yZ zxqGpQTo~euIJ(fZe*wCB^ex>?t7ET(nhWWOFYZz@a_yC=$yTpJNR}u+7YM%LpD^Sd zh@XhC95wrxO8#w9UhxcsCyS(<9n+`9srh*(mt|h8XZJJq6KS%uO~%P@@`H=4DaJaz zb4ySb@mjAF152PA1dX2dU2&@*H%R|DdPt1`bN>TIJr>Bq<5CfRN0#@Bd%yaBf&9Tf zbh{icGDtot5N=qbtZ_%4APmDPHy+b-F-%}{LxEWkzarR`=+Exb-~V3$u}otet;Hfi z=0zQA#-yKmU6{YvBs}GCio<@yJAfW-F-hkHC+;bc)Ca5a{&$PaxcQK^HABuG!eFet zI^u2-2ne@i5QTf4_n=%;R)vk&MYPL;Cz8y_`(kflye@aRXo_&ba21lWgW+)B|0_VO zWXHL_V+u4+bT&M7S4T=~?&I!+E!YCRJT-@;W0p@XwOitBjV?1&&wLzf?!MSfg!#!e z%k)VW4(|i9crqR0qQG$IbAf`Q%pPdXav06rHSA)FVp+`NOJ5Up?AFTsB$z z4#%;}0KlGpRL)wFEEMfmk2RujBMhb$YZV+Fhu*(ATIiKV2>WNw0A=+R`85*>%DHxz6pBtX-EW zL3V-zb%rV3n&h_>PNGB4R2IJ*-FNr6NE%f({_&yZAuB(?jl1VQ#hKGR~iM`Y&ZIEzkhx)qW~6O{DYAg#1=0VzD3I^s-0QzfbB z0@uD66h_@UXLNF>Q1QEZ4~ynz{6m<(HOO?$u5bBOKys&77+)4eLnR~=d*yqgAip<> za*MY7#Ht`}3_PA?L8D8G4iE~flB(q;5ze|=(P3F7KK)@Jw+FGy#&BKZrH+pPI&*GS zlk(%}jIfqt0Mo*L*^iROJ-^EJAhC;R=iQxxQQ|frNFARLW3l3D5Ue8@i?dgNJn`Gn zB|IgfEiV;e4kE)uoJX9l5Jq9xjHg*_I9?axPvLr&G*CN)6VYW@3e-87{;m*ka!n>26p_|G2A0Pp?D8ld!_Ojt<^k5-TA1&)I*30@s0N}ZVn{5A!H{TC z^xLW^+Ij{~4h!^5wRD$Nd{uiVK&%d`R5378J$h_$S`d zvlDA%q=jSMwf*yKtfxMsXaCH?y{!%WnS#g*F-CE*NR)*SUTA8Nv)gZkdW;&PsoyRW z6&sLdEL?{~Fh^0`v0&s%`E>!z97dG3LJ;@GP6Ay5H1JjJuy-z@bo&Km9u{Oq_qRvT zKaLBajxEmnzoBP0y}a1O^f*UYmSU!uxV229n7e5(`L!^(I;e6yE`n-vm;n)zAXSH$V98T@#wP&$367jb;uO&oX>Ql_m|Wk9X%J4M`rC1tfGlpf7dHg<=d}!qwrID>1e()jH}cOMus8XS z4^4xnvfREm9G@3qcJghkYxdf>0EhuLHf$g_iz23BYx`BDF`E$s6lfIUVPO;kH-%(6 z{9LA~H%onV=AvC~He4L~#@9C~5ZesLAB0#ew2)Oy2=0X2)ESczJ#r+r*bExx&c^)Y zGCk+Em1g$Al4iTth-Eh~GL#p9baE)H1-kS}@YLq}2+KpsD4|DnuEJpTZNXkTB)4Pi z%F!VY_YvK#^pFVuPLHWJQJPIm2XgySY!+{)37{0+ED(^@_jly?B^XzU z@cWE`Xg~U6#ZDkNm|PQ4imxr2*AvY~MWkDV{*0PXA!fG~C;?KtS46l{Bg9QlUK)=J zbD`j)w6rq{08vKDrq2YR5j#6)#9Tc)HAb5(V)_NZb+lRBuDC?t>0|VRI|{aRj7mwT zeY{YLgJA<#M2n#O_~N(P!|{nOlh&!n{vt^_z>X$v&%Ys9lasx%-YoxyMIVD*Cb(L; z5({VZi`j-%M@1g_7YH(6WLbh56juB}AeHK1d%Wcw=hi!p2i6`UC=_2B!z1X+A{k&b zY1ZOQQI-?ghY*-$afz^kWXy~=N`$#j*46!Uo&PAe2gSuA%)y}ww=l-z0w_rzD!av7 zJ0&}vO1ys?2)7oSFE%oq<&S*2z%UY8fW1PWF^VWXN&pjpK-dl=)4*l=4QH`*k|DCts#X3P-ko{Na|#KEG?1&r(ErEs#4+{M%(b!2nW zc3{+tePeuA&+f7e%2QGjF9`PMuxTg!e+nb?q3uDP{h*xx#$rVw)JYa7c-1hhG1hD@ z(8br@R4cwMN=Va}ri6XV%toyF+F}Q1Mgy}FA(fUFj|+4KSPLpT&Tu>{Y-*?{siQG1 zfi^M9oNUqt?RZsbt(KY6j-(i{tL_Nyy5M52DJ6vXOT>(7V#mgPX=N%|g3ztDERj(h zEp)msgzg}FnD}z}Gb=)rsqvVQ+?eC_L9}P_lbs8Qos@1cjx#<3WZ}T53%f`SD=!JO z#4G6IKQrkSUo5{1zq;`;scOZlqWvLa{iPVyhm{b4gBW0|@v@*?OSXEc8yxvruq-xo z9a4Qqi1XH=TihVpVv@X3*?Y)F`Q%-J5pXL6za`2A;EDvTtYh})3W(hx=K$3>QxI9_ zgYNi?Xbn95oW6d3Hwdj76cZ+fMA0LekJ%v8@Z5*q63NWBfxygXMsjKu zNACfYnTk1la!`0!n3FLDBMAh95=8AfwwT4?#vY%C@^~|vnEq`6&IP9z^e9ZJj|p>` zaxlKT_XHu|nnpD;Mz_5E7l2$$bW_;B4HGM(BG9=dpCyQs!E-0>n9N#=$0_dL6%r4xU6#kaXbyF0qZTBfK=1{B**bDAh zVj3V>HgeC_7fI(1nwBdJHOvk*BKICxuvY zIlF4c#-b^_Vuhx_lKW(UQHGAkwPjZeKmeNyxCQgcOki+{&7_E&7yH7Py&rUL3}zID zx~sqB6xv&9P>cPY7qZfh7e!d`P_|>+gi#3V%mYAOi*B7Jv5#H?>T6W725K8}lQO!Y z5R&KRe&wr}(`dMjg9J?tg*3(xlR0;+2z9}G=t;3iv>O&r6=r>TMYZEm!PLB0^X?i4 z7VqGBQ;jc|bhFLxBciiBX&)_x*TrQTt`+W$H$@{%v$>?^Ej^oARk@i|W9NfFvcSXn zYFsFQ?+?`Qg4z7bCb$`>5|4;5ldyAZ)Zykcg2>PIBRclo>R=G($2eAP#Et^UPe+My zYbFRyQ;&N^I6wAxbZ+EMkU1>Z=85DHoj|hemGo-~^LY8Noc(ViCM&teKinw<_Q zGG|Z78nqc`3!)@g9oOSY5e(x}HBjsa4uyn~nzquhIX(D~!WbFYG}0y3{|c0wkCUxR zoLth}crqNf9menEj<&fH8&x38=MnC0=I7oLA$~Fj{&Bew7a1uf$Njtgg=$+G>z;Tb z)4jSbf~H!ohuW417Np=1+hm&F@`;sDYMT@v;ari0l8@^K8&(fb$(^! z5MZ8g{GjlKYxftKaA67h$n!FQT_`F;W|+qWctUS796nc)M0Shg2l$l|v8o{GiikY+ zDE8DR2BBR827dGo94rhu3CC#3hJG#4E)j#UqB348gdEd>*!?1$14>O)ADFic7jH12 zXhHXY5PEQ95^diI&t?j-j&bY%Rvo~zGeUKxD7Pg|l+FFF7nnr>%lLd+@vKl+rmw=Q zA8bHpc}z-@I|VpFOa2$@jzYUC1MIdDxXpz4YseVhWO$jyF#_QyU0R0YYaI?r;XCHc zj%j+%eMn>Z^f72yL65D(=S5gVdWjd5E%P;&Vukc-K!yZ}ECU>3^J zU7G4yEg)Ao*~$^Gk^aBMIH=3dNsiWCcTRx#liDvQ1HPU~Fb~T~HAS0gGx(Zde?`k` z-C(oId>~4%>o8(SL}nIch&S5MQ&}?ws7tK6z%MZCU^*+prD0{vX=t3Z1cEw-Uh_*K zuB%qh)%d3Xc&mdNl=v$g;S8~3s>H@3TtX^6u22Def{+c{>t8L(MH-WJhZiXdT0mly90N+;qq4``PlVs2FU2a_##xTfHqhw+j1`TCDk1XE0go zAB(j!!F4zTv9X9!=qBR=0l8SLmKgFIadCcw)lDPjokRkE9~DFDMNL62Hsk#`X8vil z@h8D~UCD4cM(Y!^vlUMWFFFG*I`NVaH%_h2=1NB=)mZstP*)2p)K+XF8a7~e#e^t= zCQA3%$BczVQjGl;#4km=kb}DJoM`3ioC4&Ejn^l!ed7!33Uloljal_k zIkarrL?|tX8<})jr;Bv?b(W79ZTAfjL~&l%-1xqp-A`m9&Ver?bfrLlJh@eCZ2Pe= zvxiVc=moUfCi_=~Q!_<%YR7WlgmO(R$msF4`=qayo%EUHJ;VnUneWqY&Bwzt%>~KEnj6 zqgE^yk-Hs(7A4vI%mR^e6k)zl2)ydUu;6w*XSO6iTx9irVX8h$)`9rNSx!AZI1t-^ zi)V9|4WmpT7-shtD521Z`ArcnJUU;1a|O6S9Li`|{+^J`PgW7c!-^|%7X7W(Xf)2| zcd@R&%|h-PQO;0XNba8bHh@d4$%X(EBXPnxU@jL14j6jL7~(#G{!X)j2_`j5kPDNJ zu+fJpk&$kzbM879#2>`2fq7@8^8hT7avjo*rqp6(p)Lll{6o4EIF2ttHO6PGLv++1Wq>DpPWCDA9U4U6YbLgD;Jt3CJ9*zg&;0vMs z^%TlY{pp*+s01R5ZtF%-&VyC1Opg-cJPwxve3kDOc{mh5>fgD-{1&!uM3Iljg}887 zqvE*pv;aQCp*d~;;zdxdvKI?dH@zaz^&3RpQu!XFn~M{a#&|C}7lI!Ponjz?v+?FJ z=9C1ZN_)vEG>M|ig+KQl$mi7#Eu&lEo0KZ+(8c7!{cSzF3RR3?lG6PNVHA%A z$ozO+1jR$Ei%C&ljP7wMAm7hUHyGc1O^9n@^Ac^VPcA{cY*`G+=Rbk40%r*{)vqWC zWz$_Jl7Q7OgQl$<)0l&qE&&t@@<3oBjGkS63i_XiTE!zwwI(LZk1o256J;r0P2(y|~c9~Z$cLj)Bm{m_b zHSt)iBa~_({)~oiiL%IH`$s(wyAr@hHOxpX9_6Z}P3z-V@xxRDZ~?f!u+G(BW+rn@ za-!*r?jHkMBDt+To)pRVbR}O;yele8A_}_{5qphmz&w2yM%b*-HV|wE%-1G*P_#r- zo(FO2PoP|IR#M37de;`YFwr}{Cl&mXV1E!5wPujvXbUqdaS}{}*X4r5m1IzNn<#&p zz2iW9aS^mR$su}+tDLmj`uufeHsdYYuNhw!l)t8L$>He`VP+C5KPjs@cbG5GIZ9L+ zQzcZF3gx@(P~{&Nn_ds?N>V2V&yxkXV4R{3#ZPX4c1?&+pUmtx{wa{<0F9*vqvwVA zCAOfw(RX9<5(WSm*fH`5$_+_$B+5uvd0wC^BWn>hox|~ViB(#xSu~}sbW@QZQzl^+ z$D+H0I6qDXiMI&yCs0Y~E?8P(zncN6p!9voCvMlD0Z`U_Q%5WiojS{mSos#7T{8t3 zVHbIz5LcfL!CkA(7%UTE!buGAIN??jxf*l>8je&6b74>|$U6@3^pK98&EEMK; z>Q+pgCRQ)~af47wsUYQ=#r`N7={dYoV-dH`ZBQWKPd!`X4^ub`N>yC>rAJ9%_v z*ukV-SFCpzn7>PwsA1zQAzp1tn3`?e=;hRThj4!jl__Jl93MOH2BhL$Gp5HvJ?Fm1 z`hC8wAHOYBf|Iu?ajgJG9o>@-NB)?9r?iTGQj1-G&F`=!UBY?FJrLA|@fbGWE^2DX z(>}q$xVS3%UVfk!D4%F1xvMM?l1AJfoa97k%otNI+vqo@SmjWy{Y04oX!6;A#o~y$zPpHezvY3NMVyoX@KvFnJ zG}jPBzMKq=ARTeEXmgbs6ILK@6heaxXdU(+5tMM8a4j4CfoFd}_7C!~J4+CS#wkhL zz<-If+%`FhMaM^Uya&L{V@1EJ#Q}dTfM$w%u-t4Ic9(0FLiMni2H+&c(i!*NN zi2HUz+~nFa$5%_5ecMpnk!hCD>1b`QM@a8UzCyI!?#ci+>hK z5~bO$#zz+yFG+pQOI<>;^l{`9?Z$#6g}>#v zYr`i&;UGJ~YFr~CXNBG;=>j%<3QYZY=B(IF&t^oM7>Lh_r1Z%mOEb`+e|FkF?42Ic zvs)0pGVppNzWlGid~b4`rN#bFLzr(w5kebZiIyN20j7;4gVpPVxs`B#W=9ln{LQI2 z{-{@bbqx$>%g=yOILxod>57+$G;5RUHLexloVmJ?cuS}Qo)+e@T)nMQ+-q^uv;2@{ zB}Im+#@_^)@2HQvqwBdMKXl0~6A<&+!pvhrx1jlxy`E8L#{4^hzIaI}?URwkt@W;J z*-ZdMq`}raX0nL*mG2vZa0CY*a<+{Dpi_kRTAS2*? zdd_*$n{hLx*l*?pyv3qWwa0o=q`*|6GC`nJY3D!kj?2cydTk<=KiY z0drwwnk{9(kl;MftCcnJH{!3#*o{QM(jlUNRO4x3)Hq!y6)%YLe5X?$2k9o&w3mxF zbt0qYjO&G21}bBNxa@u4?**V(Db4tjfGiZqTUwRB?-elEHNrKDbM=h44iTm_wiB_$ zqC`$Y68q2O7dmE4i`Vq*YV{IIHah^X_$nx}i3Uc-IV=1F(hWq+z@}-xl;X9p1V)#3 zic(o1SoF&>Vcpk2C|M6m$o2K?c9~!e!S0%oYCnONG?<@mYW&qdf#4p(fmh{fuS2;9 zV-p&~B3=Ckx2{waV&>n%oUu+L`xX$_CFZ0mQVr*Yw<{hGscb7Bgk>h- zWZsU;mB=E-N&HyLhh*=TuM%j=s)`T1M@kE4$Ly{-aiM6nl&mer;$#0;{ESrM&MMli zjbk0V<2XYIjWtFL{NL)C3Mjf?CH^HUiyNz|WEXn$QoB#R&ncrWxL;;z5Em8JumRE) z3bDOVvo;$UGq&tdCQa*3d#fKMGjhiXAunNQk+FROuv0M^pb& zI*W)6kkr`-gIVUT^9tE=B*-O7k^VK1t*{(#=Tg$klY8l>h50)u*a?x>ka5R;Wl99c z=PJX(Tm_;jas<+j6+Q^$V)yBW&9PYJLm=?4M)cFMctjLctatasoy+qKJ2-_SB$^1n zNDZ`8|DO=VGF6LF=6x%eX6f9aTU9?QI`drCg_YPqh--slmJ1ljWlLf17ucgA8NC8% zWX&bW05dGLo4F#OI6t#1_SbXnlN_|SqF;bNG>MLwTGUHK*FK@xIZ}w3#`Z;C#A5F) zWxDEEavZx$h*_3YdVG8Nm5NtcEF|}T2IWr+r9^`$@bA^LON1H&MevIm;L0CGJ{pem zS0;tKFyn12{wmrSCC{3kICEq+;3}Ecn1m+%-%r=q7{|HluWieQdFRY%aNyW$=_&8MlxGWZX;?#`b zC?U-T=l1^-;ym$#vyavEHNdF7mJ(>PW4kp?)0Y((EIuPzQ;zN#q>V=fQCGrUAucrE;93TFrLLohXbM1D<`HDypZqf_Gk8y)JBu#mG}kTMxhuK`B@ZsJ)I62s=|a*|<Ro8n&sLR06P8ef?T9%Mz?>1vh?W^y`706%y4>)3z(#!%*=RC#HOe1tHwjMLs6dhj0tU2HW8TEBj7 zo!Vp*P^RAbJ@L98U3Z0h;Y+J*3gWz_Fs0uT#Y7kdB|KH!t7p|w*JCcVS@AlGV(w0h z4+_aeQ(QoeM5}KB=Gw^CxD^KpV6p>9MW4%v-z$liB&A3v8Sun&EyUlo4fD6>D z%x`E-Dfu1jD6+vSzmFN}NFljBu**(ckzWXNZd%^9($>j-plx*KScHxTX25m@?mbO|vpbD+n00PZscN!Au!$FDC6_2kQe(Inrb zDXsAK<{c=C(*vT)USz@n)!U_GNAjE`|UTj^S6GgGKT!Q7@ z_KT?R3~ILba@|#Gx=~^Nz7(YLa6VFy#iDPdyFY$d(j#*J82=2vo4Z_Z&xG`Q0d9W9 z$|`hnAKRtKTcLK;sV@2~P!=&Ph-FS7B!6vucr?DaD>S*Yvi64JasjjjbGokdS(JZg z(xor61^S%;4R|x=#AAB45OH)zcohxMZwmA`y{CxnK35`Wf@tBd3!!}mSk}ciMDRro zJgfnKB+@M6OfoGWci*jmIEqVAguW%nT;ZGo9pJSBs2GjJwu>&fdyxR6bLNK=1-P&Z z`RX&hlLXR^#8c+D=(0@N8K!Fwexs&}y0nv?EqLj3O$tkdC!x9it$wt+lAeDE)Wph) zmX*SS&qJCWj0{o_yeq^NGaWY05aa?-!70CkMwt*l zA0p6UJ|T!w&tuQO^cQ$GlL+v|DsDo6CQR*j_S?%TJ7;#pH}y;@YbdB%I-8DH^RM$d zJ7a^r`PGsvZ!re`c5E%o1=1bO&@iC8)r#U%`h^w6=wKYEN0+-P z|1&ku>B4e%(ySKemjJmij4r(|Df}_+UsAY)6@<8;SVysMuf*DdGHbanp%N>9sd$Uc z8A_)JA^tj{*-;oF4c!M6uPIJ_J022X5yI0;+0Pa!4aHG!@o7JzJhs7B8_iV-?yfAdg!dpT*p-K*JVfuM&rdaN)b*phmv)1v!21KreSb9|nz7 zd}>atp+|nIFt)c*7M>xR3Uy7N7VB1cc21n+5fZ2#8bHdC z@j)9i4(sScBi*131)S$-uX>h9aSqqwOZ5y_FlojKg&J>O1)x;2zfthRtwd)5t8g_} z>@6S*p^k=0Gb)G-$E)Kpp5JE(K}Ft(--xD+jN`=np?Ac6LM)Fc_PKSCebVVQFn=XQ zVvKVHI1v}B%b2YAza%GeqSUZwz<0Z zg*dO=(pTij29(|5So}f|4a)AYBl`P`-?gpP^7w}XK;lnx3|E4I7$1a&}@L~50D%b_HdmDr-#=FrJ5CXPjOfI5;HbihWObq zKf_n-Br#@+4~{_cg;7SFb@eIywr-tQa^>V(N|`qjHeCiZex1Ar2L! zJ^72BtKOyv=o_@`X_e?+`BaOb{@pwrD zA~3IOcD#Q+&n{GjC}y(#Vif3TgSt?%+0$lmNsQJ*7fZSsN@r$Bomk_$-AlYGrM>3Q z9}45^1g>qwZ6c@~F#_o`{qaKnk)22_R$5SG$W|y>C2uGQ{&Q}pA?G-euDjwnQTGdl zP@Rrh)8ZpXI59S>qpSlzn@Pr$CNt8FBGpsmM1PM6uX$DJu_=3n2ZdXnF#lv>zVVR& zW(I;{NjXlK6UlHB^H~AvR(O|QbQI6Ih{Nn@N8$rVgHQ*RW4tHU5a}F;6o5!VG$M=) zy0{o1jwzD&va!8DWX_MVmrdlALb8Bhy&a!9)-R8Yj6{Ea?&PAqwx0dn9#%}}iF7|; zZB*-(V`BVOU~X%=DQ5+EY#tBhb|ug(8#kOC1X)n%&#X(Cx!X&ktOJN6pIpCET3(lL zEmNlsC7vvEB!(KXuv+n~AXkg)MuF(pLR-<1 zlZy01DC?pwnvT^<>-*Eb28BrV%d+%B5oViiv#d1w<2pgP0@NEne?n!Q3})d;YM3}; zAplvj8bZR)6vfBqP4A3r^*kkK6Sx?^B!JRmZJD-4?~3O8tZTV@=@bZ87K6+N5lTKO z#FfS9U?9ds=Y}QbhU$96l1bRSE-I7MdRQ-`az6D`-l7updo;C3Aid%=Va}O-;ZR&6 z+9jn!V13ec8i0kZk~BNgGn^yPC6ql7?EJY9RYN_l&?5b8FED^EUp zW|1v13nZ)6=o02JibY9_?g`WAO>=}JFl?MqS|5-pT{YZ~Uy6TI7C0T81x3kO8w^BE zM8OhWSHVRN!pu_kJQMiENGLWv+pl4rU5{-G+Zems1hwG(H_tLbB}A+_v61q_Uu*LSs+4r2v05T^mxwAHQ5b&^kjD_v!pRZ^rD{ z_*`&HHpbFpcv@m1RToI&4o)IY$j=oHFIPAp+;vyb>|~DpNnw*z9J(IuUr}uSS)kuh zKoVy2-+vdxT&N7OQjFOb6-g>1%rZl<%=bXdPOd^7(Am}Ug7^{^29>y01O=bqnD7}r zJ3AbhFymY2`%Z~{_DFnE1ivD~tQ+Wu8w;SG+5-E)-Dht|PI}yJ1+>&2sDrv{_LR>~p0>&g_G4oO|ReDZW z?4{?-AhtD{0Q&^NAnXD#U>8A+*<8{tr{td|weP-Igxi!}jHI4*836hA(8XWYbN&(Q z!@gMhhXCZNa6a*45w2>hHouixm77igT@)5>$vqua5+R?#SNn)8OYXzt6fp#tjn|+T(-Uv#LeI6Zq(xUqO&lOfl!)p4w1U^z2=uPTgw650;j) zCD_?=2+4`a*95@-WYUYv3lwFfBd0RoDmv$aLVwDF`whWf)@?*cyVb6Noa#S>9ZXA@ zwM(!vIiz6+>HV!h^MHY&5)YR&&hoW*RkSN9haGnFYhDXt=}s4dvO=M;HxTNkWb7ay zd@BwSB%xz~JVMW|B_^0?$gW=GALn&-#va!d?-D5^d0)>LWFh8MmzV`uaW53+Jerjj z3xg*FxNKYtz<3l-l?Y3Pev|+&3ZW1ar2p9Uex8SU>ad>SazCn|R=n>9C z&5Bu`Fv@_<2SeE|qFg;@NX%P*D8QL_<0e3osl z;zjgx@;cu3=ODEC42*n+^z6@%>Lw-@fR%VppueR%P8m4=@(U0OHLvIpB!AeDzRb7Grc74Hos^#ke^<{+Ws&oEQ+oV!dk zbZ@~qnZP^|`z=#ruVuhte|soFH^ zCPap+1M#Agxd9k&P|Xj;o%a-wO8uTM791-d#qmX{Us5)iR@Qi!_!mCAxWq~qjGeg~Z;pH=yAY$1TE z=(wA^Vnmt$=#J!h^Qqs1P&wjn4aIRHNQ~JXs<{7>*s0kLF9trsn;iQxd{nCpqSw z@CUz`B8$a0MZs2r&BuB97nVy%#2S`DAv@VF1Mfh8s zoAmVeph=3=1m=#$P_0>VEkWuX>{+(dbMES>DB}Rp60mWm;y6PTLo0SEI@G>Sr1_zX zKj}3$3vmf`uGbeg{1HHTcg&7E^qj?4tCL3jpAaXn;`>pJr5^;~18fr&c5yF}REKVj z8d?NJlhRW|#_T^qPf2XL)G2ll#Me6J&W)2E^7DA#KrGaAt~5bTn2BTchk=TN3wX!+ zjqSxJ1oOoq8I0~E!UY~=eq!TveTkr>CQxJCBg9!VVPFM?(b!ZTidMDPBhG`BGCF&) z4SU z!~xOt^OBG}-Jq^a8?LkgdHJ#8Z6Y2KY!l0HLGEZ6pQKAb<2_-PXqFPJ3Yo)Narxur z%j^>*c=rf$1*ESfe#;X8F0M)!A=<^on}QLzC)O84)6K(dchx70pEa~j?2T382)9Ipr&Cmk{QI%2zPug zg58?zI^};ubWdTfESoIN3@me&dm7MuWh=_5+dcy5|MX^t|8qsTxLB4a!#=+0zZ0ZbLH4)Ax z#Vq7TtGHhnLNk*Y`KIT1cF_qF2QQu$;1{$_C15?CBhQ1m?=_n!f<^irosNveR|UX) zMq&<(Qoto4lr#k*V1%3bBD5EXBtj;1yvnlrm%=k2h#JNag&5pjf?Aqbx%E&9X6dG2 zYre$m=B$i?BpW?~U?Tb`ELqZcuH1kqLvHg^+o(%~_~-~h*5X|e&UuvC z0HtwU_7;%)u{>_lgjkKW-ky4kqCWLM2o3a{W;~;3S1qlgQ#9dr?D`IH9s-HuM_+zY zh(E>JBP~T=5>yU#s2b8;4ePuMO!l2~I$}c4a0Ocs%^^pMr0@i|8;`Gvph0x)IyY8G z15V_4f&Pq6Z`DRWEg?n`LWyB?{vN65XJ|7>aG5B-gVKpbA?F~1%mtZDF+XYEU-$n& z6YtpiZITJ(Wp?bD2|4M-kO)^EHJPT3lbr&xU>alQwBfU38QoNd09Lq1`9N2K`>cH4Xw8aIgds zDtE@ZKGxp*|10y;*%YfXaP?0o=tO3)0WI%vh zbu=D7i)@Vtq5fXwU{Wb7n$1@%^GtjEp&@O-X3nuk6)At?2)6% z=75z*;6{`6e+3Jh5kVB6UY`X1f=sheA$-z!Tq`*+m&bQ+S4+b7vdivm}315G?4I9puuhK$v6LUSm8Es|%s(Ow`EnQKHOYxs%2{ zq9yclO?abxnV3s*E&TMV{0yfQkQjM*Qe@6uQyxodA+$4^TTMRISoOmI@U4SsV4G9&->FrJPg{-Gg_sS!x?qE$ z>jNJJ%FIQ}oVvxJFlx)UQ}B$tL{lAPQei}wSieFE++DgzUHbj2^$dhVsWn!H!l8OQ?m1jQ1z6V z(_-Isd8Xo=&T2p2*dB3+h99Ig_q)dw5xG!kDS5bkO9%Wb=&%nYI_upnic{mOb^ zE-yZlGFZG^h`5i7`0w;=zA$C6#bJ)oSpEs%%z5lDS$qocY}7X{2Ntas&$<>^#L;*@tfuovxMZv&6DJ31vztkF$p@z z`lcC|2vz17#Kj&PI-hiYnzr)a6+-no2_d%cMm!^lDYkhtVj86R!%-W z+okxo2y@}+``9J?o2_ns=`BI;UbFq>X!6)Mg+AR8($D>Z|Y>Sh8>HuMfpp; zI%TTGa+`rzj1%?8cTppAjq~dU4^8y;HF_^JXUao|6 zjfD5c`QXpBQ!(VrG#La~4*&~z^rfEIersSuEp^p{B2fN7{`eAw{ebRe(epq&SSQCz zA0H$9|I-iDyf#7k=WY#=hkg##5W}<_-DwFJOO=Yk@Y7Om zMZ!9JX2t00Gw6gI_ee>HON0`-0^~NpbxPX{aVxVmS6r3&T$x*=JNd~|(d|A%I;w?X zV1ImU7YHgqOe_v84-}PqhR!X1BHABj-BiVkZzMMVESNu|I3+UUyHrrFoh_#TpDWWb ziYMpHopvQ1jft&JoRy!)7&dOwbMd~8C@JPGIROFc;)9S&&^L4C#?Fx5n)_MGABlj+An$ zFjBxGs^7C0lpC6!iM5PgJ5eBf?cw>X{5%hf))(|_jJ>EQTk0dS;S zTH^)#K&UimH7jwkD3>?w+u&eaB5bPe1MEi9dj9iY1a+zEnB?kku^9<8d+|JDe;+3) z8J#03WLfgOC_1+|Mk;Kt;uS$IUz@5m<6i%UlWDc010Fb&PNe4Z-fE zltkK%<}NBj1{s3WMN?vPy}~Jey*(1g>V2RjH$?4VS9X%1sd1LfE*I^F(PT6hD;)~J zOhT(Fdgl%z{X54;aNs`zU@j*km3ZG*AY8z7Cz7Rl4WVWMdM%vMV_1lD>FFP9H{)3W zE-XtGro5qO99E=Z35Zen>fJ>)EHpHR9VLXf`Z(Ib+N&QaIbV>Ofei{Fnxz__G@LsY zyEPPij8eBNt=)w<*MOhWXeAfx-{p)Mau~{4LSjuR$W$r_|9ff_5FGfI?p5lkvd z)yCUtlU_)=*o=;jSgdE44iAkcM|hTZG=!JB&qd33RK#JpAudNaYPjKoJmK3KAnCTyZw0^W)U~Be4-G@lN^UAhT9{ zpiW|QcY)pMP7>k~2ktt+-7UhXdagU(f6^BU_ zdmQf9J7;pNp=Xzl4Mro*6K%oN@_@NDO;nS9Mj&@5_ooekCmxRNXjWPP8mP(4Yovrld>G_MdOpf;>k;>v9( z*CquZ;~1nKn+c??IV9?gt3(t_~J1z=OG(_N9(xIhruVi-xGM~;A^HD__@ zT3yfnL|Ppsd(`;?ox^1IRr{%+sohtvwg>it+~!Dd7of?iC=GQ9%LQ=2+SpymT+{AG zbpSy=aYf9dWjwevg%R11+%dV*R8T^XObY((kt&M(s?#|II|$4i4a(W0WD6* zBHRe<(6FH%6F{}+A;iy=?`m@3*L{xWSFxx)iiJ~Ml=B|q5_p#0>m37<8P!i`O*^Cq z1ZI(8XQ(pocq|ZK>73OOV|sQjIt#!U?>HeIDaLynags=j54Y|zpyD)vyNFP_$kg%H z@5@vuyi)(5mvtZKR7B7omnB=j=vl+H|Un}jtGBx0lUc6j)!!8#t3{^ zPkXTqPbff!Km}Fc1Y}&Gdn}!&5+{i^W3}HLj$aALLRcfxgVw$KpIBf%31V*jt}s|U zf-z++t`=qHNR_|>?bY%fRFkGJCB&z$QA|r)g;GC*Y zh}lX2DKbx2ubf<>U@S$_bequ3Hr?rhxmSYVRPQ+EGh=Z|nSW&*w+8}q3gPQ)%Hljd z7k!=h0*k%|KrKgE<27Qbg~iW|kld^|ARtNMtawJxv=5en6S3SW5NxQ&4v&xN(M{D| z8I0>hsu>laa{aGExNazCZ4FS3YlY@+h*v&#c}-((w{E?;NNI9O(ZUVI%K}_=jGzgP zQ;AmuxlwR4N&%El1+kp)J*G!|c;edvoeZ6XG)OC)R^){OFS)mVM3BGF_N*NXMNgHH zs5%-cuMtMo*!0bf+ePH5pfQTM&sf^5-I4Q2vD}zV9uQ5j^Q9)T6Wi?@KzSHuQDLQf zY<)T?Eus6K$Kn=I{`?@3tdNNwoW}}`5t##JmIX^-+7e8tmIUzTGB-NUE|L*xB83NmF>}8Ss`7Qr zjCp#7W%kK^;2bD_37eq0?gMyRki~(WM?X@77tHPFI)~1gbEEe>o?Rxb z;Q@kMoGKy@A8ZQ;Vz~`VTqR1q^M(ixeS#btUxPb98 zD;6U<>C2kU}?l-+7Zqc)g%A%fCGCI7Umzd5z*tS>WDPdO75H8!KzSy}lO5Es=))BDp(D5Y55JfqHVM z??O@b>C>mh{ELbV6+BDV*#Ak8g@9cY8<@0^s(ugDm87u-I4EsL!pvVS+X!jE_I^+a z-3&6_V#XLQ5nk-4ub^VYeW+O3y zwDBZ-&agnL+u79-l^^l!+V~8n8H-AYF^+fey%l6W5NgR$PS(2|jM6!{)cT^FkL>1T z>~p3dOA{05D5JinNB)#Dvuxd8^oW$*Fs z5-+mbmzJ^fC(d5wA^eZ({A)qZOOX)M=DF)yFwdC7-Mw+CNVBCE<$%WTdVGHoVD4B9 zFkEq&6zRVd3?opAvht>hm$?qW`D64zgiprggk_oP!Ad$-zrOf_cYbQxrvGGXvH$i1-VmM^$9S8gj$TJU9NxI{n zDTZD!s2|Kaf@Ibe_n(NhhFvqXFuKgqC$K4bZXvXq=-tL*NLg4_rDo*T7>1W5{3q%c zIyr|qJ@K#zN~OCfP)KTCcq0Ekh~GhU-OBIg5Th2ubiZ_)dk7^9BIk9*10u|rzOe?{ zsy%K4po?~Obwz)Et`B0Rt7j)fA3V?(j|wnLuyJK^UyZNa?sw_-QPlaj7 zn=v=u*0Xwb|2U@4^UR|hX(;v@d}lZ|7ix(?4Udfy>)f;?IYmF@PJ{tY zU9EJEFy~5^EkTO~IDGQq1@<^lb&g)qON>)7VU^j`xwS4MaA z#G}zE$dyVLn_$kyvhjC9VIc9&8JxGc2TDTGF)b$bY*w`2g2%Zq4(-xQbzAVtuLxx(^jjlu^F550S#K#GO0DF(5B73JDU zv5t7h5x)hRYB?+q6pj(^*RusXGaXM8E|ae=66lg(hlO&TLFL2u6$xc~Nej`VrwR0T zIJQqmPKW;v$ebk!PM&jrU!=iaGM0!6`GQR z1iDygwdf~wiNl1^0P{L$#^?V~B|Q(2 zBscuH4{7;&pU^zJqI8U(il(9j?_pAb8Qp&bP*G;ii47j~kFsBZgSdEQ0IdNUV)%KX z0HvNWE1uUg{aOQPno!~+e*%F+(h$bxBKWdm0qO+kP|;IuHi8hTAukYU=HSJ^^)k%7 z*9mi5$lIzNca(|Pi`3Hr=&cVqcj@@$Lvhx_&=jnSS=3W{Mx;?XXbSo8BajFg>w4zd zK2cNSJbF|GO-W2-4KHi3{cXd?5L-l5sg5%#J&&zy+0RFWrV)Z;J&^f*O&9xiCWrjeZEX5+0ojj zuY~d)0`dQ-Iu9sIi{kA|kf0#3Kr+HENRr)|*+dW}=LIAR0{70`$uoDDJ7Mnv5=1}| zR0Jdk0VN2EC@MKi6p##mat;zCDe?O~{q(%=e&@`Y^VV}aR;TLf>gt;X`&&KHY{K7pVZrKnj;GC0Mi>gT_5| zP_UC6(GwHWARZ#j?2@RCp~SI5oC(u#y2Jg9Ae3gk{jt(Zd{P}t_VG=T?r>FICYvI< zC=SOa+Na+u(43YeND-wedP1mk=4HhO8|EiVge^33MNEE~FfK7Z`+O_x6k#qOHpR*A z_W4)mNoq_7jO)Ax;x=Ip$2NI00cMGvdNb4aH{vqEUIMWniJW*N^W>gpo@uk#CDzO?NwnG64_JgsEM5&%y>_@t6i-$F=&nT-T+Y# zK~2Lx^21CvYU#$yM{gG4@h8gb$??HkAkIzB!_r3JBViMK$IV@mw7>Q?Ahp2;vl6pJ zkV`M3A?wwvBArO3Q!i(L?fi~|Fs3JAE4BH%1=&}x#uXx6a84!kaMpb96=~#qV7SE; z=7MNn)UNZ3_;wjkM_s&SH4=LXb2D~ul&PbqPC?EAPtP5kN{+q{;%3x#xg*y90Kmn@ z=et|~G-344Nv$#ZVG$m8nRK$-1u&f3M=P=6|MT{4k<(&`p(S?FtLOdzZ$tcAhpMbfpxUfuy z*#m;8Ytp!&C?APmWkfToF0Au%O&QJ2jP|Jv)IQ3sis4x5Qxc;1EXC{-wh=`ub6pBQ z88I%>oiI<`h#&|I{}Qfs1XloG{fuwdgVm@<8es`ETM4G~ATw8Ddl9gBtSe5^r<(*P z1IQ;Y3drLT_a*6$L>UH64ztW+pBG^mG2J~F1?>0*kQPtJNnxuEEV;xcC z!%sY{*)bf`1yG>mKxUQ*b6NG~Y7>_Ygt_ZsAm8CzzUR#c=xi`6;Cwe;6hhsZa$>#t zi(e7VwPWuV0y|mZ+4$cg3U;cd97{eXlBm6v@%TjF6bE(<#-a;AmK1iE2GhxMXvg zO!hZWWQWtsNV>|ijUcK*_o{L=@oJG2Uzg0{ZV_e@uOzY0iHFMs29l8zFN~DXTzZjI z28(^U$R>qCO58_~M@eIF6zfLOPP&(BAy|$Hfy0$Ka#w!cLQooSQ)WiX!v1dMd&Nch zcJoDF?1UXNeUut^z@>j|{DTb~pE-%P+?9&>&nP~Nyw3X`@vaPAN(}VEuC1gU^;>~uF zP1j!>Tw?}v)ObP!#YVDBx9d0invTFt^eW=a)RZF$ix6a7>*wpgn5FnI+a3tX0xs-oimRvO~I@8Y_w*6;6l| zkQslQ3vg+xwn^HmL|CP&tSc7@QGcxu!iDf?VC2;;(-l)ZJFo8QsM0 z{|;RN+^p!5Z%56MKNRLx;>0Umc{@Rni=fFqz7XvqbE>JMjr&(DA~m}4G={7RVS59PVOEPJTcqKghE2Xy8`pz#G+XYS$H)d zsy2#vT+z3ggZpflT@&DP$$A{ZB6An&j>hUxnVZ^J(tWQNWENp< zoVKXQ(*F?Z;xX&UvPAXFA!1^u;igyX4Wpqf3)nYs%npFF=i{4@~MCKq@x2#~{cw=pc8sf~m_D#NBdQ7glE(j0TUB@x!%5Pbl2r~%NNi66SiM1!tNZi5E+V=P26cNa`IyRH6&bbbOqNezK0G*5#KxIJxtf zp0QFNSXy}Q7O-aYP9aYfx;avd)mt1LrTxsfS%eFLVc48lrLCaZwbf%Y(QX8thz<8tb>pQg zb`)+66NeIG>id@L(+?gM9G_uy<)g<0yC&9q4R&BaolsYKkS5n*V1jZ#)`dG##gJXO zIUKWNl0@4}XbAT%tXT0wL2e}}mn5FX(Zb9UroiseI7fifXRDC)Z~s>&R_T|e7%vo; z5FQhxVCQ)#qPinhYxp=Bj-980&=T!3-Tz)Dc!|3wP7*=0w9T3v@8=(|XKTh1)A_*+ zAF7P?$9bZiDEkuWtge^=;>x1pk@7A+669J?FZMiF%q+qnr=ig9h))EO<1mIP^;yNw zjoL^u-n~SSnL|S+!>hQw42CfeFQX5~A4&*6Hq0`$xJ8gT#w5zDI>LtRE@7UA&h69W}ro~wK zyO3_S-sFO4Z2>NojN!TwYNnKs+>vk%Ikpf)O|e43QP)6xCYn?`Fb}5Ut85OL`KS3= zv;PLdC{j({aN+<_L>TVo@OU8ZDJk+zIiVJ6Qd6$4zK+R(bB{dA(G>2HDsQ&&3(>g%NCE6mZYV?aV(b_*w<$uj%a`a+oww)(BTh)4OxLpxUOdn2;YMw4ZIrGARj}6{S0w>0c zjsoKiAzH!o_*?vMp5Hl6p=hgaU&N@eahI~DMA#&1|F`H4gmELFaO0NSe+8)PC)2}I zcPxHRi$EN_6M(tfjgD11wFPztqLJo~;-X`=2zL!ij#h<(#cgMUxnrrr=Tp?}gyxi( z--hV`^FX+o8`PA$r$Xh9id(m2nR}=(ml<^?$7LUs0OJyE;EKBt!n|TaQ?GbJh?_&= zsM@J7o)+lotbtr{@UA8+df}-%ev?U@=ILV9BH!cJ%qh-^=EhS3TmfGM8IE_#FgT$@ z!dYi`!oVB0RrNSSgzJJ%PCl}maiuUP!chaW!<=~`{4OTL#J?+)=DlgtW0yVnb~*Go zPd#=OKpt}X9mk7MNG|iQv!_GmTSDxPGr%N(9*$F_Hv|y6bxK<-x0gfnBXbm^<1Yf- zY;ut?XzzcY2qib9+QC?AZ-|Md4gu#|qWQ7fL<%`rgbPZ4^X8Um&fMdv> zYP?ti)P?d5?Ng+W>#|xrD%w*wFQMY^R$g^0vD){6-Q38B@&vPi5DFr#Lmad(gxlU8 zj5WuO37lvQT{>lUI!PEJPdm0XvKYi{H3>a&vVLGtp!3AM`K&Qnu=If)E!<2c~t>N{}0{+QU^Rc3sR07b~Dq9pzAbDYUN73DWvJ zi6Xh`jWR(FRM^-K#|;M(m0C)w!Y&ch{EC6}$g&bYBm>0}%$ij~h;!Ox$4#Q#Us$I` z;ttW~NPj&&+xV^!HC#LIpDxx8k&CLsFkG*a=}tmXQ}vHWL!X+gFwAMj?Hy3&IcMEG z_R7xZO=0DMEb=B!AH=UO^n>|@g$*YGxF@ZZ5FBT`0ad7Nr3xkV<6NepTJ#G+@ntVp|_44iFvO!L! zZEUQ_?H!tuam6Q-HcU=W&^Ni)^uqlY zBAkX!=hQ{WZppqPJ6mc=6j)6l@#e}G(~cq>4=;K+_DUjKzfky98N`gB8C&%`1`g#1 zVh<6H!M3u!4?7^X<9$M1F-ft=TkMVR6-c|1D3)sH4??+@P(C>aj$;ZPxf^1TS$?Lqg|CS!&7xw_p zpKCgliB*NstNVCWwWob<98iYg*%$5<)3?qLHZk^je-+PGCDM`H(Z%8i5r#JELr3#Z zeS2x*$TYScb{6z2rirn*py8iq`FW#%iS)Q?uT@9m!Vzd%iu*mh!Y~Svi(K#P!``|c z`wBBB@G8K0N4ku$pHOoGC#GD!D%csYS?VO*#vNJX$;^)FKIj;oC}x-uf#G& zd%_^M-A10=&Bq^61>HzO~tV4rh1vE$|d>?*{Obj?-T;P1~9hy$#lM*KjCE8Wh) z_)wfQPY|7>4aV9>7AZ?Z$u{jDLUNO!w{OOa0?Y=aA|@u9eW4!}fzs{hSXhADZElaQ zEy~zs$r42J&QF5wE|BIQW&N(}8_sbW-i%F;f`BWXcz@(Brzq!IS956R^@ksWxza3^ z_^ja7aY4>EnQJo*^clpCinaCauo&Iw!q=b>GSLM{olj#lu+ zeo43qX$Ug`Sv+weq%T&`4^)<;BP@YM+W)55f5j1 z_(c%y>frzvHzycBjR_+o)Qt6bSA_eG^r<5;`&a(8Pp+g+I0+&*Uw68~cC`>1cO#5; zTTgaSTtsk5Q3NH{{H&`i*NM&v)AvW>DFH4@r=FkI^Dq0J0_OA)5e5+;J4%G1S3~qT zu|$xtBvAfJh?5>1mHzZP0dxY5C=_#_{2G!@pqDXtH~ds6vx+@Xy2z9C)B6ZFi-*w5 zMP0O0ttX4UHBJX`fi>YLkF0l;SQ3>m_K(%iAe<*1lqgMk*ScGfy9n3Ty5;_W9(+%U zA|WNgAF(i@6rO8?oY}^M3S^dyCHEHbj1XAfisSaz^zEsOb7(4KS#iNLfnD|yZW(vP zx&mA#?5&a4hU0EQj#|}y15Tf1$?~aS*Aiz2Y|2p&FLM@{S%Z^~{#Z`5Q%jwd+eYUK za*jOUG1QDp1;F%lbvpiB{#+#+eBR}sr>@_|%D2YZ&HxDqok*>qkx8-D=b?K2PU`8t zsm>lYI0wL`M4QQILY&yP#4v~BXo!Vkm-C8n(lFD@`eKg~OK*)xVREuCxICc@X^;1X zXjI5^$;rQg@L+BPahw8}ljuuTDb_m#emdc)`pNugj$+s_7?%s7fwB74!SS1!#M!O3P#a%Xghv=< zd5P_Xdkt&>we- zbXV-J^mWHeqMZ{Hgs(V1r)I;jS$UQq;T;0`1<674E;uU1!j4;SJV zuBIo~&lBLBdMhwq4z=GXqv7FHcPUo;Q<>0cKbwu%RtOVDFYVeM$BJ@Ubo+!Yb1ZeG z18~%Xfb)VVm!xJ7iL?F;0^?dR;`^??`4uL!Xh_bvPYC?!!B;JbFBPV0CyJNc-Nb9jQHyXI!`<&v)POrg^;S=d1BZ+C6XeH^LXdi|Hij#;9Hpp zwt}2!-(Y=?wRgJ+bfO$1ps$X-g_wimn9la8?;KG^VGeC zK%B`3ZL3^efXU?_Z)Nw#K?2=DkXY(*a~Vjc@641;K%bW&f!e!M51f4sF*P&ln(*Ty zU_cG)0|xOX*Fu_)tVEct?k9j&<@yTK!|S5V0D5$PGfCtNT?c4JbvL^(5voOk9FET@ zXT7T{ekj^i(W3s)vM3!f5u-Ui_;kzb@7IYS6ZwajRZSM}{? zN~#AY9k{u`?Vx5Cw{UfxcO*6}G4@U98VR?VF!Q7jBegicq{q4T9nXq(BQhs3>c8?2 z04g?I(?f9Sux&d*ba0&$v##`uEZB^uP_&EUn_^U8-8;a{GYZW?g_hm*1UfJ5beOjLuNV^Z#taZX$>(A`2-`r!7snsDcbt3!^qn8j~USYs7LXWe&-n}p@A zQ0cC4S{hFaq8iNfc!ZCZe?r1B3TQSU1y(JQM>0T^OxjTEPBVLJirkv-JE|5M!}|PN@>h+z;gj z#rZu?zLhjym^$O&2l(A$GFAaey3kUv$AiEW809Dm`xq7JoOu~ej)CG!4;N5&IvN`Y za+7mK3TY#@7eX5&^h}Et9)WOC2RlE&{e1v3An6NzU^dTJ}XjJyA@NJQHtmX8CU2AP;%sJf16@KGbw z4AjXHp8Gjf9W?AR$c}*N@XUcK5AEiVFD{lH;8x2q;DD(|5m9G1LMZ*t%v$kj#H!tkOQzIYjI zVhTmU8k1fDU|=A&%kjdHNdJzylK01t%)hg!OMtpqBvsm|b!yx*@r^696JG_V;+zO@ z=Kp{w=PZeiX^KmNuYtMJ_?kcykS3CZy6u=PhOnGWy*$C@1-a^;^!fV;(O(E!rk6BfG8@v#DO?20|pAZr+V!;NK<4jM70Z??-GM)KKet1QwE7?IzkjWo{ zxSMjEfr9)A0TX;cpWhMZe^dmK;=Mm^7GO5WG9LelB((W|i%_zsL}w|0gfJ33Oc8py85fUcqN$_C7eu5{ zUY8~3H$QPeO!J#@oCv!AIDLZyK2fe2GrtbJW5rJ$nXWQ8jF0@!9Jy^H=IEBsA>0tW z_*|*R;Q~mF&WX~n@kQ}7Dh6hT(*@8`W?)*%oL_D zlmJLZyL@Tq38v`knyd^MPYHE$bW)yV`5DZeaXjc{lah9CN=uM_r z|I0=4sz}qO(zQj@~E2J2&*c=n!iT z=|XZWz&6u}dCG}Dw&cq#L@u;za`io1L|#17qowIS%PvBRGr;}WGOHJs>amv)l209+j&Pu8 zbA`2h90yDS+#m>b%;pk#o)%~Zb0U%s)ITpHO4r^W8+^qnr-_##|DI@fF(xPc%#zzT zz6$2@v`m>IA_Vys4$NmCfIvW!lz8#Z)mN9gX6b*kDplPQ*)EU2Dyhsqw)0N~(@)2RK ziChZM<^LzdB~Ozv3vqu;`x>CD$ILSpeWG2CdKWwE__+Ww!k}{??q34JtH-fmyY0Ojap?rz2x8DNU_ zHl%BrB8f(24(f8z&Xt~qIuS|wxMjgSM)2S@;dbUxp^`-qkWLSO_cDpTr*Fz=_x$m> zNVv&HVrrbSoWsq;3F&A0&fTg$$Z9i8ffMNLb$FW8&tH@Qb;2o8xE6;k?_}9n^u#$L z+=Lz2Yc}H&0l5}vY0|dm*sp_8-WIgD(^lZyT@2kvZ3g%wKG_0QDt1Zt6*XT(Pg?%B)5!;*Ta3Z!^1<1K`q{*f57+VcQ z9GTV<_vzbhj3al{0QesmiNV#&IFjGF1pEyUcUJiaOUel8y4L`9elpHd*?R@KK9rKX zX<&1NxeT~tl*Z{KA@0KRm=q6|389EfuM<8Y$VJOeo|M$vWkC7~Be_2oT9be-8&}*L zynmT2z5glLWkcHNV*ZSG1i8b|49q=klK@-{{7Uu3Hzz~8Y0G0vOYG~c#ZRt-OdW9r zdA5+;*z%c{jHs?!8_-$spd~xSe+bCy6kTL^NW#=`Ec?v@W)q)!o`li_=R%{Wkz~EV zI)uy_qg7#HT0@8f4IwXxExQYHF4zcT_~ka&a(D73FrSRAoiL zH1R)SPIv&X71EG?XFVWKgyWda{841C9t#Q95bFbwDB3EHtq$4%D&;qQM%*pJ4Jcm# zIP{YglTdT5zn6P{GIKQO!mtU%1dg3r3g{|fMS>Gx+Lkijr&JAVy6Z>u5CfL}7O=Ub zXCW~JJwk{pLyeP}{VxT%^i>{t^u|1B}UMb(+n1=&1V66HsYCfP1m|>IS}K*-CKq@#W+T!<8peO zMX#Rx-frvM^cNZd1W?K?inKO^(jW95yCpCzd>aZD3Je@jObahE7fq_d{QEBba_ zBp|XS6+mHZRvDj(ba#}q(!8{uzA>oVOYU7z-Ng4dDH5t;bIjAr0^B>%R3V2kDqr6e zjGU*nOpU9*UHr~PQCo(6Tw-%;s3xS%O;~PxX|*U=Ef(C2$Od~^#DLR{6hQ1(yT)Q= z{q8PU>tLw-Jk#tlHR;5kMZ4NqKyZ2Q9RVI7n02Myum;F7-vM^@Sr`)gK&J?G>P>vF z##GU+s*b05W)QXCcfnve4ry`l_-9dhSaRsto_B03Zvjj(^tg`H=Fu&{;h10OZ`R|k zz9n$xCG$(V_nENV25d3th_OQ%jEeT4cIc143(M@Ip?nYK8iDRWJPX6R6nBK#4o zYlOIupdB2ImqoiL(XnKxq$%O`GSploub8qup(gBz3O83|Fjqgg8Iz3GBOiJR^n_Bp z$E@`*;m(VF8pjgp+TT1$pbDXL$z)<45CfmVF6>;o;$US)*)^@|%x>8oip=SeH4bZ# z0e>&hvsQbpJ#NV)+az5mUVcY@)%4Ac+1dJbXV7C~>ijVx%elRpi!&3%-l5W7C^FK zFs9R^O_hSnD1#aqRmGMi#x@DtCbo0i33F^+h~+^=nZoWa)GT5Hg9;6$bP2KACGGxU zf~W^h&MNWK@^|GXne@*kB>Qd5y5j~xx%2iUhxS9e7Fpq4P?kn`(!EJNe6}`P1zqq+4$w+p$??Icv!|7pqvEn*`6rB^rsj=Jc5X_%G zhPhP~b)UF<&jp|P_H+Op&T()nDr>;gO|%Bb#!{H3A120~-U~?k#jhy1M~+^`!nq5> z5f#{)!d*)?)~e-tLh|H-G6(M~0^AH}aXFzJj&=7c!k|@QB7R1IGu6#VoT+O+`GdVd zQ~hSlh;|XV2V>Mq%RIUd2sQ5In1X)#eFsJw9*fB$T!=CDU_CKK0EIw-8aiAa_^P{!GO)Dv^lc#%?SVokZY*QTyP&KfN7@PFLh&?-A0EE529~=$<3iS} z-1X@!l82T2#dShR9<8tk_4^q>HKEC|9{Y3^iC5{l9Ef7rCCo#B=Wv(R_a*%#- zuGsppnU*7*c|h*VF^F)IX3)cQ)`s}DnWH2p_Rq-%$97q#2{%VNx$28R_LdNOUq4Q2 zKhd|j!>*Md&+9INGZ(T0iuHSpVC%-zh;}i~>~#VyjE#%*?F7I0Cv9~ zuE`5al7td!4sy*Jt(@kH^%OJ@oe9q{ZNl8^^fV|=eq!S?MMkSU!)eQ~qfiFBERE4U z)}b_ZdAaWzeN)%|9yIhf4?#F%NrSfddQhO(!b!L~{LVp;ZVV1Obq=vzi44i-Mn`;K z$V6w9-j@Y@Oqh8jbFO+EDj;Xp#9kR&sDZdlpoX+=R9#y}>10pSANLINtJ{L}jz)YS z+8suwIJoz=RoBPDoiv*0I(I`TR07Q`HqNY9{jqF=aIht`FXO#e z-)^K%Zu_y~YJDCSYCh<$Jwvb|#JRF?Nbo1u1UgqKbtSB9J>qkH+gFa(-zmhT`jB zAeu2XYrGXfjH ze`zYcShx#k0}&JzWl%YsFOJ?nDwE}a4;dAacBR9J?#_j;bu>eh1UNlT!scRV9yLKZ zYi^`6DN!Rt%1edAqcM5g)Hj`sRW-eQz1`tOGOXb!%sQf1ko)0)c4Uhl0qyyYIaq>F zQVlO9m|!^SOTJ+*$h402D;R*kA=*8MiIx0f*B^nn4qYfd2jfWrn!ILBjjtWWw_6vM z@rsh>)R7XjK0=HBKLzGw@YKLfydML&z_Lq-ABc9N_oD7OMx;Az=?)++Jlc_1XV~yI z;tA0%uHRdXE01xAv=3*7={OeJCG<{R8tB&qI+~_g8fL?vfVg-p1NcQ=^ruCT>Tr6E zWV(>tQIj$fx1%M*_~FiDJ$4r4a-$od+Z_9$#X$|bSS=eTmdvk8>5|3F9Eh1 z>@?>886jYEFUP4nT51$`OMWfXjtwY__$=)<5Ul%wy{0=u(F|YyF}CWy8Y43Be@QG?OpZt5 zIMH;U!~~2aL^20W)A95zC;B(!BkCE?iF6s#9L!p>%Sk{k0}o-T{ZA3%K)M>CLM(W) zlR+x$8|{moM7mA!`b&?VEr23*k2Z7^^jKjkzC4+*&x=8mpppqGVe8sVL*SMzu4)LHSqzRgz7a;xci(Aj}M~$zzS4vFlkyR%tEP!?uf@ z4d(I;SL)oOxk>}F91>Zk{V$+E)i)ipwDAne?kB)%=^Ce zcuj<(V+$8c*L2aNL&*g)mrIykmd%K!Q;BNf<3N4*97+* zv?TG)7v_==(G-$aZW82zjbR?$A3I)D#KRSQ9~bC4Vq6%x%XS2R8W%&kv<-}~(2CuW z5jhG-?~cW@LR=&{SYXe?1hDWWfSCzc)nm>mz~$+a&l$QozJE&$)Pa2Z^WU$cN7Gxfw)72IXrbb2Uj0vn#M^- zj4NHn?=CFXaSV=cWPtR$>m68_%oIQ?FrSTfcSgTR4^xcD*+iwq{8*u`B1}NQuf$IT zW!|tPSb6+AfmEkUs(^b$n8S=i`f@x{CPPOa6F1)%3M&JC_5v-=eU z(RMmB-~l92PI;)&WKoLEg)si`9};`&+jUNFipJ3bsA89Pk~n-837eA>dHlCy`r}JM zm*@xQtMe$PnfS9H_)BrqqbXmAcCFc5utaF9-S~q8vG>-q74fiW^AI&>dax16bl*P~ zQMF6bwVlDJ3xr{8t8g_T!Uad2LGQRyfD;+ zfBU2`wS{aozx5{wSb$nctM|V$iJ7C`1A0j0#6-${1szc_`q#M<*p<}*3}zZA^y>mS zZ&|M;PSvLi!i=uHVf?F%!fT_L@~d>umEl-+SlowWn?DoP1(woK&ydCUgt_{703zOx z1W;%xk9i8=((-pk6Ye?h65#yxz_=E#@B9Ubeqy`CQBkf}x@^GSR#Ngqf}K%KRvxkL zRgNW3r(>~+sN8S8*6k&PjJi0mzzGCK7jcRpctgcid_Ny@xj-k3I5#{L2_T|8w##ap z%xmn#g-&w-dzSsKgK%O=CrmFtSrEU=Z+YA-0&Zac#vPtX*F(81gNS^U zI9PytAlnxiT`-GoaYF%u$m9*SCeTeifbfI)9H&<(+RLzD>46cayTuxrJ}nM{k&7@!Szt3Uj~K)Q&{B(5)cmw?wjG9!~;rTlS2$)&36M+e>?ZwjEeOkF$-9sd<+e#n`Qrn>uM zP3b-{lXa|qH<%|~{EXvBf%OG*4WYC=!UJ;8H|{BNlM6gL`jP(x$$deGi44kf1(|u= z45P75xevf)?qlN>qxVBQcaA7g1c5F6K#>kRoOEAi)e>SFW^DJz3_(u1(qBcfd5!>g ze;J@n+ywYn8Loen#|h)%66nB0w5)CWJr*D2 z9|w8Lf%z=ndeA>s^|0gt4?(y-a_5I*-exTFFqrv*8?E#}_3?s8kduH~eEAUw4Tjdq zF`{o*B5gi+s7rJD5|0*1pk82!?TB3k<&H~EmFO1WtmXJG*?b%-%vIweh+aKCqC_g( z(?V|lX_*kly-aQw2yhS8B?PqTbn&Gg19T;N>$nJ`|E>FYkzs>}JF%qdh`WVpFk;77 zdxCHJOGlmSCh>(R*NiDr+OKpr^pz(ap>@ivSXtjhK-bRxdp}VV<4m(gHTp}4$wgf( zZp`spFd^Fc|D4t|p8Gsi-=sCH7kA>Hq8NjnY?_$r-WTZ_R{H14evFhp73cyMiWZLB z66|7dnKL~LnjpGIIwpoUpO?w$_<>~(@C$*Chm8l@;^A2EsWMx=Z(Eg#?!p2cuDdqG zj6E1jln~N>8UQy6%JTu$W@vpR#LbP1B*b|}*CS7Zd313@4u=%a6lv3I=cf2+54|7h&c#BNki2C0{R6V=Q6Y$<0nd6uvT-edKi_==|8h^3(U<@K5M0`r^kT za(BzF2A6yjh>CGqr`r>;n`pA*Pze&o{xAn%f7D&+#`ak>{R6s-_sjkx(iLH)8gAkO zoI?NmEl{^7F8b%uyrv&qo8J1|T4%hakREvGX?PqD*D?&Cw*{LM1C?Ga1nu!3VJ`gu z*QiDt?eX6-m@Y%5r$iQiyNHTWla`Q53<=94i1AJdf0jda;Iv&AnWd^aTbMH#uZ$!m z{BO&M@`lYB!j10~Nug@c;{2EpxZkCF1pj&01oJ>WrvC?m39%h_pkw+b6dv(wvBG;f ziZ&ygh%gHqxIkn6(c#BuLd~f@67Gwo-v^+j^u`_&VEkCL18`)+)rflqI2}3ZNirKg z!i2g%ewR7fh{y33tdAa?7-9Im5aEl0`>|OzsdivBE@qxZ` zTQmo`jGmT^$^Qd&CJk9s$8sM7xDwNN}D9+A$zQ31H03 zA7K)}Ii#Vyd{4Z+N5~IzF{tifrN&IW*nH%|+{om0Po@Aw=&v{B zt$WdhiEJjK$K()J`RmY;9TSh|6IcBe9UN_$oK&eMen-vR-nBX?cIFH!J8NzZkaehz6 zHwvMfXjvT{?vF`J79p63wN1WFi1SIdeH7--rGQ*Qj_I|*C)~nI1Clux9qTbgggcOo zBeR7EySlF|!w()I+FLLkF}`>0Aqx@D@I6hJV&WqNCzrEgb>;n1!NEGVc1I!m4(r#|b;tDO9l96Grt|gf*5LRcX{utquY);HY=*EHRzokp zLJ^R8Mg|;IM3~Et)f6cqzBDyEv5>|>=@xjD^LCaxX@g}PAklN zE6)S9r+w-7R{^4xxn)l3BXN{yxj~*2E3C?=Q>007>bsTzidB<4zaNO89{HXIv+P={ zfjGlntkBfL>j-k$*`FX-=n!phf#!hr7VI_l6=E3yeGF0zBAn#efik)zEEY*Pf1yBU zgY6c_aD#EPAUaYT4>&CS4ZfWvvviXCy5h7F#Hg3O0;jTZuRyBKz9jhpS!fMts)xOH zPmGFiA=$u-<-c9CNVvU!v});OgGqoAvXgPk2cq1`&~)PCh*f>&WU$N?-Gk*cKS7x* zT=-$*z;H{T<6_du#*p1X!aP_xzd{T{3X$2Zl5|kGFSn4DI7XPui<4D3?7N}Nn0K+@ zYe(EB$Qfhrh(1j9>Rua=YIh9t>fKMjX@boS8cGpr;t8Xh_>B;%+m@WYZoUqbtJaz3 zQH;+85nyP9*7Zgfz7Df(k4L9TVz=v+2nv8xd0f%3~Y%Xbq-_vpq0^7-ZOHt?qVT_VgKofU&E z^W`Pbm!1N6rUbai#kTf;0u=C#Fvt{a(>7Jq@O1B^nz5 zQTpWPT)L__>RUx1Zr-IW)OR-of}^NoF}2)5l#9ws&an6%0px>(qZijd4O42aSV$Eq+lij+U{K*6a@8IXH7%qOfS2Mcqh5CQ3( z!*Qq}7YNtOmEL;XB?L~$91=M-UJy-(V@1RZXbXfZgn28&dtU)rgp=VweK4K8{YkKM zl9w2*4quc}c>}YPIlh%JwDYu<$?>7S;eyp7aY`E`d2+jhCm5zdxM{f4H5hM-mT*&J z*%^%arkj*$G)jiuMY=-F15#)nk`XDZQCEWeFh^ROiP$sUwPNw@~d_v-qYeC|dQ{ z!hc1YnTS@@Y{>cty7Dw4>gscrF+NQdz@i>Wl8*aVQQX3m%r zb2jChmaFvhetDBKuahz`h&^ah53TWShagM$5Zmo}|PID@*P3`dW^ zxK_1FpqIH(uDQ0eO-*KK3HM}TCkkm^zOqb}8$xMH|GOY}T&^z+#?_)7v0C$i9OL2| zq2^2#UsP=7uPc+{=vUn!t{0S>m#rq(R*+3@6bMJCZIyQhUKh!rfr%7;%OceVF4gJ3 zD{T#8p6baj&cqsVvoPn4bc3-#th5b?%Z&255}S(7A#j^TdABP=3{<$zx4&qo+eHXm zvG>PW!f0Due#QRV@@+O^_9;E#;DF@OA>p}4jOYSgTqVSL(+Yhtc{^y=ZwSv*dy8~N z43{RY&O?bM=JOM7G95#$o$5Qg``${=!YkGL0$&L)U+;($p~!(&2nzraIB4e_@l-V^A4*4JcY z%thZYXw-#5Uh9@9yTC3*dOXg6BSh{3ftbgfHQ99RRAEeC+VjkfS424(?i|R7JMP{U z$cq9u6&SuB3xHLlb4TOK-S~!8oSPx@e-8pKaD*f0mD6`GB6i}UzZTmFFb}vYfF(gm zeq@~71<{qW1&dhCVejmuZU=Jtq{H+icXtc?0BVBCXm)jgaQGDD5um0-HXh%E@tvIV+a4SWfua_!s>mPd^=qsw#@ zG*{TC#5FbO%o4doOeQDGcLFj-Td4gC`ZlX|`I|PrUWoaDI}@g~*UKQ1IvHRe+8`{O^dd~=!7aj3SVJ;r8*3j^b*B9;V=%+!2y%+JmFwmyUvDxrS#0?cKoiPDWFbB3}*^=EqT&`*U#hH5|Phd zmfi(=in1`5vsc&CQA-OnGrAD3`(t$hW;oZkDzU$43d`xd_Czwi|LSZP*%d(4*Uq9dWQ=Cxn|Bgu%E#h}LC(U=!D`Qh^Segen3G_#kv>go4rZK2+$P*v zle6xX&kI#+&2+!;PKC;YjcXP0Kp6}3u3oYA89{EMs;};?Q3o+ohI-gY$I?Ru!0`=N zZ+c2#pw`pd6Hf|YRzrD-BEIz?hsR}-U%TB>fa@!x*F&ASeE6A-gjy!_cpY%3I31;?~f%n7n^zGgFr)^frpnyQf6 zV28%UHUgXr`%R`ShTt!9AO#IV%L@<^cI(8xyy^DV6AP_j2> zid|k=+*r8lfe6YRMfC3qa|0tkup`k0>zxI<@Y*JDsq7XZ6caZfsq?IO2sD*Jmo;4H z_BwUJPP9!Zp*R9UQC>6cbR>Q<+is%}8PRdva6BNy9fu+@d2D$ofJ>S5hw7`p5av>H z+KGHCaK#@2lAEmKw65$b+9{!gM~pj1fHOn%lMwcBjzX*9o_{Fb6m8~l$e7mIBt&XSZbR%JIWvqF0qg6YPJ&!0*WCpy8 zBpaB7k`<>>EitYj6YW@)<~fclV=+&$dBujKBhD*Py;{qOrA}kkIsM9nmV%T()_FCgY6-rb6=Y$G8afPTu)FP2XmB z6;Cw7ah3qH8xdIQl52%HOU#Viszj>EbUBv$F=5;oGCE2-=5>X6aB2G#w~40s?d|RHiN2kt{GZbDUp?BPCrs#f z6Ak~c@{oq;L6NR6j?i(|`I-P%1)DKW#l)`Tj{$W1%VSt&DA`0L)QsSIPI@2s1O+ri zYwIPixK)sgLT>JMCpiSMK}ZwSn9)7jf}koPGpDLfl<{Z zC|4J6Y^W2CEMsEc#;GX^l(@Bw$&nI^Vtd?^W9GX!G}l5uaRG3k*dL3FATMNIYLS#k z9X|!ioWhc&Cx%Oag8~c!-YNlj$)@fT0Td525^ZCY$#Eb|^l0P8aTO}ct3V$+^Sgcq z2@CquG5?ttSXH&I$3KLcArdGwjP4TTr6)bh3Ey-n{oXQuYCko7 ze=K|gfn6gcaDdGPOvsnaW0?1*jP@5!50kgOrGCkGDkSd(|5Zfh36te;+$6vS&09CQ zT)0)J1{mf8|EF*FUWCx0IOxP837jTVtA_-T0YkYfe)%g1rzgiKGI^l`&3{sX%D7N^ zc{XPY2zA|c?IuY-B=Kdz=2Anw4XAG(DDO1>Dk%3jnvPYHx9T1c=;~GaRJM0d2B2*C zsThuDe+?lyrVAsZSpHNfmz2|c^vj8J-9iZ?FPl8mBnm#X%xaC*PUD-T&=_&WO9UN- zTMaCLa%Z-ikcm~T!PB6O!tb75VA{az%xPXs{#Vl_p)D0I*SH7kb0ok7xq8e+TxFzy zv27U+AvgKi_(_=s$|G^B62B2fsi#kw8t?1d9HGacd06m_A}`i8ytfU;qJkWz%CR*^ zy(NH=7i|2hfV>#u7A{$kE-B0<)UkwR!Cwe;XPGNIy8o1Phlc5@XYxBmlgIbJ=^Iuf z9*xY6+eA4g4?E(}LNtZafM#{UDj`R0;nnlc^zHKG(?Q(up5^3NRM(sp7mD&kqsKmJ z{YB3PF_YPyr)upgh~Yd?o1=U0Tb=`HJ`5lg@IF8ZaLI$=v&^}UKpxfZq1Z_z5p?e$ z+36iAfIh;-@Ui&Hc@Pu<-v#Irq{f{g#2KJu;<%{)0stzVPUn~nR=&_tNS~!p^TQ@W zGMCwT`Q5>51(Hple2rcw!u=R033Q&c>AUT>psp?Vu<89c!xLntq2wReEE$i?gW;D) z$IsFH9pPvro?3}Z^zFpCF)d|}d`Vp=)QRib2*PN*SO(N)DW6GfaS;LC3XFO#)x`IO z(C7>XP8n9YxcHsLQrESAF2osD)8-TlGESz1sz=Y59JgHJ6r?92Ur{a!jhKYOl`eJQ zHau}G{d>OQRI+>AUj*D9$K*+~%Hg8T5abSSpm1`z@?{0aMBHDO1$MVEnl0^r5O?EB z0Tic=UVMYTT>uu)EE%GV)=1&Vt?feGxtqL685fJ5n73(XPB4-f{uklyAaiT@ zs*Gm@z%HrdhvP?ofFj?%N^d;;N4|3d>w-CQ?k}$ZQf1K+#iJt3NCY+v%#wcQdsh~* zYWPypgfI`pp#s~ViXm4(7Ngv8+e7hO0q&>Ijl)D!F8UDWSS$S*igBbH*JvHr6zN)U zI#6ev2*^E;>7!G6woYM`Uzc`ST(!hE1UcJ243wBaA1;w@`6}mn$X^KKg76F{9^yos z4RXVyW?knhAh^`pIx|kzw}aE?2V%p&7Qtl)ErIGMg5VlkNY2MZc(~CdO=;+wG5R+^ zuTz{F6o*tV2sVdNL!lH;=8wnyy+GLs(j$?wF%A~$B=C!X?=w57`+snr}qG_dMB+!g= zu7#W^emW`F?&YTIK(XRDs573@CvBV*vhji_Qe!?&^}OkNXx9j<4qg>NRYv?bZYW@w zlDrqGO%j|ZY;GwcUt!#+pcbYE2YTX1H$u8bgEedn;xYlQQKQ0q*%M3M1ma<%>7uyZ z`6uCSWqD(Y({6!Q517>!m+CvWC=vli$|wwPxfRe!OIJ-V_~vZ|(!kd=UEz-5HZy@|?*70mqa-v&LXd-yb)Y5Er(RhJLv_iQ&d( zG14xWp%UK`N;=c0w!{f{@ogSXosR3|3q?~Be1m9Xt46#pL#$46rl*x);kyY%AL&RF z><*$_dYnC^1BG1$IawTq*5e_1~rahh$?P82_JJ2fduRj22R^!%J!jfd&4TVzgPF$7Tpl`-lZ&y67Pxy!O zhEvCx|AO+EQUf7_rtA363wAcRiICv{-QvX(%y%YW!Ak;N-HysAra*)7h9GzK_PXAS ziVuY_?sOPSs~++oq+1BvTFz8T8XJe<`07La?%dfb=YzOu0?8=PEfP*I6XbMIkb`FjEJX5 zW3)$)5KkF;LAMe|32+%OdB@mBwEK?rPv~$z73TI4lat?vM$CE~)U{-^%aH)nL;@Y1 z`KH3-MstnjJh@EL9UCd81JO4JlF|Pr!eH?@)^C_yb&7PeG!V};yIdp4r9coH!XFds zQLOv~s4I;@U!C>s8X=CFRzT#lTgpI>Yn1lyb>}Y=q!lg@RfHqk>Tc}!8-i)smp1(JInuF&D^Oo83>Fz#OeU zh!1gGj|+0657p46#Q6f;P-xy|y>!HLAaJ;kn-J}BwW!1@jvc=9JcKKUf{t@0?uO9V zHNj3)cJQ?r6_C3La;I8$(HEUgJ9h=+ED`SAIF#;=`Cfun6R?rvL4fT=QYI$&POh3E zKD{j@vrqH2=CF5iBqs5}&h}XK!%cQ@0ujz;WL2J1q{ z&sun(5Necn^TTn!05>+Gl%5ArrT!;$q8_}?_s%QOE+#dx5LbT{j4spKIwOvr_-?_t zNZ*djZn_>nd<`15S7@ewt~wRJ4x!$Rk3?q{qk`w|&FcVe3i+)=sglkc?iQT;fu1)? z=5Y6wpk7fONjs5s-Y8NWulHjsaj5`Wy1R$dw?Acq1I2pWl8IR}Sf-yBk#nHjx*E#6 z;co(FMx)Eai&;V(46ltkzoQ@TtYGFMSM`{PlJ;cp5@njkpK=m@zxHp!<{1#H$ksafKQ&THh*yrPAO7MTnaUQMijUP2IEtqY1dd zijB>3Z=2!-lyic~qMWSW8NtahJC4l+I(HfEvLab{7}w!F(BlgEV9G`v1ki+dEvgG=^U5?=l(NYm1+miW9(1C!-J zJWDVwQ;^1cML~F02`A+gC7J?VTy7L{v+523j6|LM(Crp^ACipGsJF#oBFr)bDH+sr zY*S)Q86q(_V(tfo%Wc9A4)#AQgtla5(^*?A@FBFTz#5o@!_@@2BvPzl*D8Rlxw}>8 zY7>MTpZ1fTwvrhU>WpwQq>1H6LNqI~-JoL zuWum49A%@z-hKa10bDWmXz7w8FI|0B#6uR5PT(yeo>_FDfIHPR9-ChPn^m0uyI4mG z%bgq0ijSWN%&o#v3Kd-HOS>eN>rKV@g9!TS4E!%&sqYDHRk~$u_=O!7Z}dFJM|PwoGr|`Ve5c08~slSC69QBqjw8( z0fwYQ?1&W?DC5z8ISG$TONiruWVE~Yf`powpJbAbexM^pz6_c>7}AMOZsQVROfwv^ zvjzI6D92+C(HXmpl%Fr-F|5*Uig-m>9yPKxhg+|f!FZZ?6p;=02MZO!*y6HG55)!x z1Gz~jZhlDCXcMd@hr=BnXB1_&Vn5UwM`apq076|Y9v5xKj5L$qui-^N=-Jr6;0P$@ zUldZa6fVb1xLWBr0DiM8x+4eS7@-pHh%j%kDw3$m(ehWo+@ri%(K!EsK+0JkZ*c4M zZc*+J9I-IxwtN+U9#uz9p0XI<&X$F#o?p+{NvLzuEtmA>bHcox)n!ns(D1{^M<}r8 zyw+HLaR@U7!?M0OOtdp+a3C(;EWk;RaB!PEHNGIs4TZDo^<^&fm9GI6dNO9$WFk?E z>B1eI>k_&P!d7&NBCLFtaueMbfnOJ%c{A*1@P^~kB>|mRlKYqhZxS>yK493BqKT*d zGTRL=x}Ri_S!tA852NyFAJdj1B%Ew(X^C_74bz!TP;sojG}J^}4KmqbI&iJ9iMGN{ zBOOg7(7mLARt~HGgt&xs*$(;Dct|mGPaf@0CwluYQ)EC#&|!La^a@iE>O3nF?}_p} z+FPr}0?Ycxsp%fZCZe2e^4G^`7vkX6@y@t7Ae|-4YTDbpxm*!v>eSRq?+M9Wiz%`% zzO}p);o2>Vk~mVNYs{+=G6$hgufIY8rkhboy4p~nM;v`iI!iioBfc$oqL`@jar7mG zI&y@9%yC8jUG6NkIqrx*2{Ge)B*JSv+$YS5vwYT(Pk0ieBc`qhoM#fYr_8*vZQ4et zD=yc3GRiqwkkgbblYBFtEsS2TTP$&783!xvbbR=%Aa^RRl2_ySm7vWkR4Dk`zd?X` z!;X*VSTxnmS{abqG5X==?V{*jv$$RSy1sKiv0u?0Rsl(~2@~U394#{UIGyKe8*rj9 z_0OrTvG%Hbdql}VHC?usNOOzuG<>Jcj6GI^aH%W0oOg_9v#3WZCONAdiF1WIE5s#) zD$07Eu-t3uf6}jBA;>u^gBQHvcngrHS)BB-Y}ju!3y%B@`TH|CrwvT9cOWgWvlD7lC zM}+4doR69pZz~|_0o8OH333_Pav;;QKS?dOm41L5J$)=8JBrB5UUQhUy4?i026!&< zv~-g&&$yFVK>kt2VTMPi^?(2m#;G$<%)Tkw1!V7*o{jlXkhv^pBt4z+<7q`89Cjr; zrPl;G9r%triJ8zY97hry($AU&KtV8Blt{U=$lP=o$S`Vl5#m9h-KM%L;?OZdJ+1ce z#2y2=gk4qkB67Xg;h=69+i7qG%Aiwafp0X>KjFyiSf%RHaRC;$7{4qt$7z-h8+3!| zDxqeE-0SetP~1}nOnMgVCs>1?5bE0C%4m$8-E(DBt{`xPHXLsYa{2VGFcT`KE&Fd& z#Dvx{?!iJ_Vx&yj!pbCW;f+CEV)=!t#xw!BY&yOp%nO1v5_!mJ!A<#giL`ZNR3#+M z!H7Ov>t}ZhMpm36lHYiyyC*jOHiT1_t2O*p#iE-5F=%actTo^6h+A=*(*@kb|5GO-udw#Gj0FBy> z@;WXM;gWMnzB^tP?G|ETkpR2yzSb)y8AP?ID$k zY}C)p1Uot2O%_28T0fiU!JF~l9YDllbj|hHbVn$cSK&M2-vV3!j(WT5@<_YnPGD{a ztSq$L$&b8Y_aNQ_%WQ>E+x>lMyA4jijh1&u(i!@371u5 z*ISoGxtJHL3(BQq8BfKSD~ueG8<^6L6lKm#LYUfqXGl0B|4H%X*+pzA%F-5ghOpcO zddaIjb}U25hy=@r!Prfh*@TB(c4!xu(RgNFM^jG-AtyvYY+^)E3ZzwCFSu~mB9nd$ zAQ5NR-OT}55V^t{+lX}f7z9`GxKl<{z9wzqY?*b48hgkAaiK^MFv*rhSz~uddVEXU|J+yzFBG7RT)c z0dF`GVl*&GEWEb?cvt0djL|q3`HZr0w22ZT5q46vE_8HrB| zqnmak()7f;WgLdR*5dws`JFG2i}< z!#I|p5~ql!xN;rK{Ku-zn#E|oOSmHFO>6b0n_J2+h{dgB;BMe=8AQk{iG|;=@AB zcof;(Zc1AE11nC8n6#N@sf&cTkTUb(CRj5b5~fZ$b#m-b<=X{jHLpZXwCAC8NQhih zBVzZi0?`d! zim{InmsLK<2Bf*_6$VdqJn?gVn z3+kbpl*m96b!kV`gg74Bdo;>~Y6x>%=uIjOou8Eu`)i=V*t||CR|9n*ClblrtuB=I zo{q1}mV=74k;>R+P^Q_!VAqID5nd88(Z^=Smxdj;tz~k2Pv886@=^W)cWXd;0g*tJ zJUJ(W%|GdfyuQ1cOJi_95MLLOJ4^Bgjp#ERs|t1J!?~HfUw6eUVNMA>$(W$M1)0x?ygCX> zBIw70-5>~xdj4jgLqN2SuoJ|klPIU2?B=ACxk^}0ol{TkvR@M9nFNuXzV?m)v!Y+G zhhxCkh}xk=C{{6+mzM=lBW54@)6UZQ+J}KUeNa~D!Pr3%Y|9=l;*X*!Bq!mRUL1co zlyhg(&!Rbe1b|D;ks|3bW1KC_sY)LsC%)&F!4M;CclmmWr5C_t>N@#IqI!tQc}TlR zr!J#f?v&!2r^M1j(_;BrKPqz6qa<<$IsGUgH$P1}5~qtc`(dQCWvd=tL`wHUNoPQi zXEk>GGC#31gx7@AcV@Lti9L@g5|WQGTIpgT?kx=e9*p)ifHyuC)a@dtVYn~ilGP>x z9aMa&4Rfr_wf-_FuEXgL;%oxVMXp}bZfrgj&ONK}+loMzV2709g{J*S6w-X`|3u#tlCu;N)``^OH2vVBPnh1S;hqv)Fo|~|q)T{4V6Kn$qSWVw65>83 z4!GLmRY5MtU=J_e#>zhj$Spi7*L`tziJ(~0t;P?8IBA^?D7U?T0p!Zd%~t$IbZ#OH zmGUi$M}Ya&(}Dj42o#T z8k96%33Z`)PDOn+hjiU^RZ7>oPF7?O1SuPF@@DMdQ+KisHt%KV`)bA~tR8A0p*+6f~D_V$cMkI39_`zz!0 zio=Asx3O{K{36|W;fHf_Oaw}8>ibTeCuKGRaioBWGRkHmZW2P5ml_S3%H<9o65%Rs0Y^^Vm@Y$}lKMsT87 ziLFF2usABMM^l9J)_rw+-Nef!gn$pDd*Yr890lv_@%TuDhkjRmY$Vpc5SpHx+>eYz zRitany#rQ_`2BA|^3vYKr!G5exeR?raOR~vOh@PM09+Q`qUdKmdiElq|HsvNfLT@) zZy!N23IYWZCF~-ZjkCK%m7E1!B;%c#J2Q9E9p+BhJ0uAr8AL=71VIszAW8;NlpvWu zIjZCwM9D$E-`j7`IpfE(56|>_yZdxkS9e!eS5@aZs^DIjp7irvPyx!uL#DR`-4 zL)uH(Bs70>m2@}R36p}H_TTFI%jfBy@tQtq&|$J(BQ>WB}B`)%^Qqm zuO*iWG{7AILX8}6HWBJIPeHHHwzMk1BaeORhs>K*uG!e!B<0*H*bRXlco+WJ>j0QS zy)~JD{Z}auT8-Da;&Y`a46)!1AZBJi8A3Z^xYXZf7cSwJ>)F4nc;AMg9b0B5)=fiS9p>HqD6ehP~6J z0$ejvK2hWFNV&Nv!!-$SyDIHg!c%f(<@4tWq74ih5}_v0LT5n8eKqzp#;Nb;33@EzH}N6{XOpY2RrR=oYYBCF8Rn38F{! z5v*G-xs8O0tT+WU@V`P_0{vhzoA~DK1tb-#7KuIo2IN+AY{kk$QdQhlRK{dc@SX1q zp)!O+-atn8#MASFAON=YLkXDgZh*Xw_G8n;W6f+>3H1k10_lk*mYIc=723`qCihbM z+wH_NW<&v`Ly*hp@>J@T?L4a2YPoqw$I9ZpT zzLy`wb~SWj8{JnFYbWe+h4*Fv^7pV8Rmax5A1F_L&WFg*o0kZY^>s)1st|uZy;c^x zJx~=p;>oo*<_Gx$r70d@IWsjUQpWd|U z93lwnReHD`IjyWoiV_U><3d4RNc1{Z+^BS3ll2Y*kELjHcto&i$^{_<7OOr~Agp|H z8oyEqpPD)w$Bz$`Fo7LD_fr94*eTOvgNKVBvFIe+(_8_xLff>Izr#6=94)yc9#)#M z;C_J(bC0lQ-og&EJ?W^kpV_ree7M{kEM|K}B%PF>l`mo58MeVFC2=;gm z2{I>fpHYvSl+GN$YR-c@0-$Z8R3+AViX>yIplxzhgL!%(nj|>9+9n6!I9e{5#Kp(WMT=Wg^BXGSZ_e$N)Q$=Mc#SRC@4%l8xA88yJMO8R?D+^XUx zJ|xHl?86^2J}a{*PKnGB8_$zPn>0r5ztrEfp9#Ww>j|Y?xV6&7dkad_TqfmkZ(j_& zNRkDcm5(=mYv~A-7VH!B1GV()(`YJgfC}usiSS%;n zhY>|zR3fi$(%ZMigRgj|dtbUoMC`mV6FVUN;Ta!+taFQ|ZjSM5xE2GoBaZ zu^DCiZ+VksUZ!w0<%@?1VVI}QnH5LtZwQr6vi+>?7YlF`p}jT}%vjzkF`|j48;Q4t zK?HVo*<8djZ#xY~^lNoYX|3_-)mAEh>*Wo@Sm9qq-nQfgue6(pdxBn#IZX)7@@*#O zF4qb$l@t|&DbLbTji3Gp*i_- z65uK^V@b-b8w8CrI8xe>;@8ah0MHW;qgS5p`LIZ1jM*9QEA5IqP>$52>mz{NRur+i z+uPw|AdAiIIyNzNSfs^!`|6j>J#Ac#LHVj6w?YpB4yA)g0?ibdQede}*$!nimC<|l za@vyPNYn(6XC+jOczHya>8#k6ImZ6a%AB-(BrQ_Qeo}lL^)yGoc35(nK+~Y7A6fi^ zPm4@CswJDuIN-m4?uy&~tYFf0RN{RBv<;VX#$O)(ow4eCyNy@QPQw+W z(u^V}C7sIF`U1(^bT(po2tlGns5?0#e~*5i*_ePe(A6LkS7R&Xg9oN>jOzW$vQUF- zGLE>talf$KYK+>n(Ofw{(D;01Lv6(4LbNQ;m=UKg;2PO_2jem&JXTto{5nn77XeKz z4EKiPG^L$s{57p=d}%>IddcG%l5fY9z&v?Fd@h-`GVK||tRCEN$OnIW1zoe?SVIN$Tg&EpG`$6ePcQe*L%AQL;C!EGhB z_adO~5CQjl;s~Wpd|7a$_g~%@<_}^C-p4$MMM1nXh~cVz3OYqYD#F>3lBI2&t&|(V zomMM*R-7xyGm4!Lf56MiJiY8w;)=z+wI)0b&N+?brU&_}?w@sQxwu&^K~~ z!=(bm0|y(;lU<5THJ%Y{e(0-77?g2Q0{vxP5Fz+_hMbQ%&%UM;c*^JIw074qY=1+w zxga;=aAjL8yJP`L6)HQojZ287H^n5|OOR_K{s=|}R|gq3nN9Q2uhYA?fmQA$e4i26xVKMHWQ+G2&Qv31NC- zDN76*CE%esX!CYlp1=J87SeRc>@6V_naRWHbwQMvHFH)R{#E`q=W3Udt}B*Wp{NCe zP^=HQ&{`3QiH4Lyn3mgCBIW5geuKu*hWB4r1~-#-5N=^y9hhK$gUiR{G!oaBSz+4- z@zNA^BpwuOVlu#4c^a__s3+toHd*mzna0|uCl*|l-{GD5yrj#Stz`Zl?c>0Q+Dw?~ zthadBj2Hb{Q7O8&j@V%}(xx3M_M|qc3UX`YFjQ+K``EKq2Nl7mOpYbi;BQyU(Lm9? zHWcFFgIKk`m|Ftv_^@UDQN)It> zeH|87tceXYZ4!kEha=p?f*psWSynbwna9F)mjHLPufqI=3pbur)ZfP$L=P0kkA%5v zIDklFdSQuRyeG5moz@}`gs<@A6^G76)+U)*7rzg#kA;ww{K6_3$3 zVg)5kA-4TilpNLAti)Kn4jPulT*5qtsBkgzlWH@8UI|e3Gjo#jOth;!ADh`SIePWC zX~DGWiFx_AX+#AY)8B+(8R?EMP2*1!ggcXDy0eKOccO`BDVl*D%S`OhDF{zpkekmM zZ^Oo4mDn(W@Cn0&`;suI+u)KS`JpX6y(lO{pmx~}P7>xmaxE>N6=;2gK#zK7|8RV) zq+7*eh&C@K%>Z%J(`!R8z?w6G+$t<2y$$au%-Q5Tr-Q(yf*4b7t?RLD3kg?@YLp1G zv6c{X1d5NgS`3tc0;SjEQUR`$MKm1`mYL-`LNnCHa42Pw$4ogi^e3Od0JHvp7eI(WMSq7oOswLY^0cASBcub z4b$Q2%IH>1n}Q3*+_Eywy`*UA5N4te^6FD_4oMG815J{n>ps7+^Kh-hop-#bq&R%W zoM>!PbO!%O^FAayFZ1wA`mN&1i%t+caUAdET&2rGElru^-O^X~f8DbwOFo4Ewb z8ah8VJ}A8E=3ul;hKdU7kmS`}1o9=l#fqN!5G6B@vi)%{GB1|e0xXlIQbi_?WrcX5 z*HJ^%lLkFjFLNeE;Al({B5_!Uy4XdjalRTBF`}!4bkQh5<;w9gdL23*P)dtQ7gs^m zZ;){FvLg*I_X*1zG{-Ww=n9>t1Pfammk=33M=8iCDw6^W@Es)Grs6m_2+|pU( zc}fLe}$TFFjSXf0zxOyEk-)4)348!%@n8(;IE<>V9Rd zrfeik@<&RH$%kHv-TE0}uDf57>R|j=h#ST#kI{(cP%OROgnXRX2x=+BePl}-QJBun zw+Et+=$+H?ZM7YWEIet;$#EdA6+|nMqS&C`SBe4+If+A*b#^4}DXG5Wj*;s5n_zQV zS93t#94roS+S@yUn_7%x@-)ZTPoUc+m1Co|8T0Q91oyPpurdfG@?iB#M>$IX%s<3s z2ZMjHlBO+(p}wSBSZxIWJpO5aJOB=Ul3xdaj3=|SbC)dx-C4C%6W~eu7uIp9wuZg@EuB-1O$;y zmJRC)$`vK=C#`BT1?FotRD0}7c?x|)Ke+KcFv}UZUe+krG_#Xn?GSog)L|pp4fEWgLI~I@al=AxBr_dDFDNVn4_k%~U_bEzu zVfNFD8uuwd9$MVJ)7}oVCDar`x=L@G#Y<&jW*H`K?XkqZ6gE+~3dRUlCIm|gbd8)e zw3=(~tygA+{S`|X5*#4V^aH^q1ib<(o8CA?KZtek9EqEhfGcM3SnbRD+YM8Q%rsp$ z*$>PgMARFwH*lOJH1m&wy2U91%p6>kqMwh`%Pa^2OwpSr0&Li^!2U%gh;~F0Us!;9 z%#o>Ip8D1JxXh&|zU%R;1Bzl8MR9byQ-FsJd9g=*Klwl)dNpm%?6^gL`)3(^5*Rsl zr~oml@E==xMJcZp)p=4Y;Q^`AGg(OSVzYC`%P{h?54pk5KAvpp>k} zmP)Zi;Cv!&>b^?454>K3qBzbJ;?}X&C%N^CGE-YU88f_4B6t+HGo-!k6f5XQBon@T zoG@3;ny zFb(SjFpl}UNzn2hR_dWRA*a~Puoyl`DQ{Y+iF7lQB)hn;7o63|R+DY9VBKl7&gmWM zgOv7k^1CRa*?9uZ>{2x9PE^mIJtWwRMcOUwM8K6*`eA%&=F?~vniD|@b#_CXkh8ss z2#`WYKB9U}LB51`r&5o{g~2o_7D`t0~4z&9VbPZY1twX#*#cN4zfBG=%w? zkFnT6ARdf%9w@5E?gHFg!bi)G60OY&gGF^*(<^2{l2vaL?CLnY4H3+3@u8wL(GWQ4 zN!WR81kmln(g~|do|+Tn_9Y_*E-5tco)GMhGvQ^?16REw&=f!bM5v?uj-z01HnRvS zCjiErLv*F%^N*uQ0!hw1Q-B+Q_8!(D|2!brWe9E2Q2Z;Wc>b43@atw#d4o&ufp|)Q zRv9kDmpYigVY;caXHJh!CHP^+?3UQz5dKc{T6@`UN_p<{7$WM_hGdSWPzcsqZ~Do` zSnf~|%|K2XO`a@{djv4HDRwokM@pI^a9i7Os~Vrc6&71shl%8oRR+>cKhX+Zdp6=7 zmG%Jfl4~8U>&HS&if&G`G6Pxaup$#{66->IO^CY1vpKW$w+mnmf(uQ1eBp2)nwTED zug7*uI){S65hgs^N=Q!qiX64__sQj*e~@K1&R5!{q=b~k#Os0}En#^&ZLM0-s_^Az(t zT|6wIoW{6}Ma+fi-Y!dXSu#`|K_02cia#j6oz{cFm@dR!?HNWXHoMG$p`Xkl4iGZl zIR+lVjL-sQwxK4w_fX6@hHMa;ah^1FPV_37g(r@9ai7xewPKVbwD0yq5Kjwi*I5Vl zEdd1~YmL7Ma2bRFj6ULT$AZ8y#1X~u>`A5kdxN(~v|I}j^W%y1mE-u^R1r-vW>Cns>lVkIf_}dl8nYt3Y z3-H%rW~q}9hRzj8i7rfVetv3Ef+>tZ@reM@hZiP(^dtUu7l~}jnnt9(5|Wp`RFzMi zMlSOv3I;CI($>1bk3qeGh>UV}Kwv=H{9^e7tFgjZWka1M^aEoB+jVo&WP4s(h+9uU zLez>n9xp2}ub*6HB$gi###F<2gjdH-C*f9iqKuR~`uaa9Al7=BSjSXBZnc!D_@8U^ z_ZKSe!91fYPCSE@Kh<6*c8%g*taK)rIf{E@&GQq_axMy`o|m za)-z{T43I4rFmgIbr?#p+pAMWFGge&a$;1WCw`Mo=BK$ym-zAb!W1xZzGmD2h0SvJ+t&9lLcn_!Nc-sTkz*V;;|_+V~+mzz{*oI zS)8TT{Y*db0W>EK`j(6Wo)P9L&wI`BoRV%C=d!fx!Vx3qfkJd5Y{5<+|H9SKNzA$k z`~f4Ze?HhaiQ1BH{)U3w^rYlZcXv|-X5PTU3KtIl&8CU(5zrAQ9 z6Zga;N;`9d5M$N&)g=IQT(2&1Ak}vEn@bJRQzi(j#!&2W8IZfdvc?H59YkLc>dLW+ z!vUd8^O{;SCSA_&X-6eg!s5Rxs%S;arN=_y@;0HFEtqueUAzNnaD3vSo?^WwcQlnZ zbIWk2ue#1JYwbgc$*Lgv>@Ot>Q*{#FMWLQ%T|s$sWzu@_`P&4!N)+ZO#}Gf}{Jy}X zb7J@Cj5~#8vT3D`rv-Rc@fZU~9yCTtKlp!t;0IHRdp(++w0YfEf|^o1^V=B>rKwM@ zAqw#w^IrwPm`!h)8VBod+R4HymuG$dRzaSZqiv%+VnGtZ67OR$u2I5$K>dcT4U)!@ z*MPYdXjHJqO-uO%w1MFs+$HHVq2@cjXmbe>@if(8X71+hie>I%A^wt1aBwL>4TPC8 zU7Q9R@nTs_rvdKJJ{DrSH;CV>{;Yc~kcVB5^YW4};0%EfiBq`D1Y*~mh7aYC7pExg z&kt5x5iuFSk?YF3YxB~PWryp5Amyy7ljCRl+eMmOw{v^Z6MNnOmbq4Uv&Sn<{+ZLK z#~u3H7h~gcTMke8Z~a0)u>)%&-1fzpHv*VE?KqMmFP-qm0)fA|GKxiRDv+dz=IT(0 zS(_DtNZ@c22hIxxn?%G^L9}TYGKu6Ctana1;+Dme*Nsj>C>s;(w%{EsH^RRuqdSS1 z#mVSTq_rks)U%WRc5bc*xFCqn^RLs1-M8AGi>6@Vr}&96D9BB*x3$1W1)3$%jmEBj zB`F4;Ju9xz-+6Q^1Z~5bqY|$R%(G7c#?k*Db&IP&@EXFx{um{-nIjpt#idGR#^MgG zJ)X+|$_&B|!XNwI>Kw>|ob}-Ps{~~Jl*e!lGTtpLFTgU4^fgC9ol~K<5zGj1s+^hN z7_Z<(j6gO4Oi_JU{n`z>G@7|ul6_uKX?IUCQrK2v?rlYd?F49Y?CP@xn)s4OVBtRr zaYqq&*=ZKNy#RE@I~1!4$aQPha-Vp~-+*!lxU=Wh5kF>Ot{RQG^vY-4St6WIM_%Cq>p>=^iq=amY-1)ZjQF&V*hw&F1q2xmVKpC#T87?gfJ@su<4?#uE3DqD8Yf z4pj6vgp=nUHu3~xz8|0%YdfCuSf)@aX!OZR?X(9-xgFf6j#27M4;7_Qoe#5#u!zQL zLfsUcWO^|2s!YVNzsFXBvsB9GUT~&B@7^A(t0=28A-aZR-LiHz&LN3Ch)~}aYF^VD z8O&n%e=mWqO0gjrR70L*{X9WE<6&~R@i=MBW8mfqaWj!D`t`s|9A4(a{#YmW<4Qzk z^pv}yMB0b-Xj?oWMARfSSc>}jv;dl;!NP1$`nfG00fPEnweHwae`kK+D~S3ELf$LX z{DrZToucUPpkVVj7f=lThL0Bc6tYU2uLRe71m}FH7JA~=#{e>u@K~i5o;gndnF#6H zpE&G7Oihe>hKTE+*&xu{8i5rCiF)CX9p8Qu+!TXvhy||U5C)4(Tc1b9V=txrfkt{y zGJYb&t-#$iMND4mDWGCHU}u`F-PRZCmf&>C{-_53P@p?Cb;`zk;35H;&AF>dBFo(q zFdm&vw_PheU3_EQgMgvitVGj;m%c9Q6v~3HfX?EupXW67C(EvnmG(EWsZNqf%=o*@ z$dz7)#d%6=su7B`8HYYg$~{CZL|#a)c*hI%0JAm`IY0&vjW|iLX;zn|4N)wFn01Gk zPk&X?l##`xEfgQld0I{8{huR`$;{PL3YK+>AZp<;Bi8ts`#i}!G@~`e;6J)V*dz$w z5w{3&EwZQ968oBc>z;m#E4^F0Wg)(HXx9p;;+Q@ zLR}Ikm1IqvAQPT#xx!mWxDH~*r<}QKIvg~((YRTdv!dRWPAP??dQ`B#jrBP<`LC8= zX=O{6vb6DP@nO~_f>T%G2PMS4ZDOv2Un@Q;3lt2k+T&zlta@lBr^V??m~u=35t!BE zJb_Sd#`LLix&DS!SdntSRn6YN)|;+i zl$|zn*YSc(niT$4j+HkG)9AGjljyDDclcyTYfvm%Vttt3q}!%--gedLJO^X6Z(*C5*!Mf?V_TNPr zJ^_7?{||tt?Ese}FDWT{O`8)7zsuj5y=8r|xY9JwPdL)g+3x|E;M_C9jC8ZD#BfsQ zdLO}Uz4u*Ef<0DSgE2*b3DgPIkv%v%KQ7Q*%sSWB(+U?c$u{}`I4`^$L#2@7u#!-B ztphtgtqch=rwsEx9DG8wQTS>CZWS^T4?=xxqKe%J6fU~j=Pdj~fvz{5X7Das>BF)C zOgFt8jtIW0P!A1Z>SQCig%H>0N0M~&a=k#)kkj^9eDo1%$X#b5Fk;JpLedn}Z7mk{ z1fLKFnOSZ*!f=lLPylsENYGvRQlF95yqYpQuF~Ibude(=ngjn^K&^Ex5pB^W$TTFx zA^w|5EnF9Bg6km2y1~q9#vx_-UInpaP>&VlUMNak{6uMYeyD;yM=gF|A{AWD(7>G&<(neY|2BAVsgzo>J-EpdmeKiIOfJ(hn}pj>pMcC)tiK0J`%- z^>)^f!8l%!_;=RqSZRL$J%=F3=jd-!Kv%;Ox9HcT1&Rt}GKRDkI|=f#))IT?1P7LS z?D0i@F}V<;`G+=@;TjNZ+Vo4MeX7#q$7^IznapT4{KJCa?g5%enM`pay0QdeE0hW- zJ`B2BY*bBM8egq^uGGP&kryPjeq|wW^A8gZ#}KqK1H#zVd*tRHH!Msl@1S-kd{qcN z93uimEU_pF=rwcJAJA~vO4Py_d3)lFt z1QA)=txow!AS}&cn!^k_;AOvLKuc_j%(9UH*NzH`H^pKzA>$*43}y-9G@-e}(g=vV zFBC>QrJZeyyObKA5h&H5$aY@_qLZ_DGVU$?O;Z{eyC-v=U6urJFXU&RUeSnq1(J*Q zA$Mcfbzhka?`dKHXr?|Z(7ZA=-YbjfJxweWm-4Tbc5D`QR3i5emqYH}*ZK;`IP<2E z4$&ou#Q_rOWtnG`q+cALSg225nw04QKgnrS-1iHiG!(1fT6(yxdl`UoQtL@bEehXS zwkXYMPLiBl94-{f7qoTN;(4VqtMEJ{*ZQ9eF>Q3aX+e9T0Q3;x7Y`S*Y<{=I(iNeU z6e}%9zB~|3MOEvK)xHX(*@ph(0 zpke~ubzN_zHRNs;7Zc1v)pCy#nfhFAvVoys<5^n4?jO<06(?ZY${?O=c!i>HRT|37 zKqK>W{q2@Z>!mPb7p&s!=`jqpfIlkj3cIV=wPRDYDu~<5C39Bf+#t}|rp`zK%1-|p z2rM?nd(OB@wyZ|d9pW}^v|ZsN#)Q$776jg(=f7vSOo;{705`YKio=xf=aShl7M7iH zra(6wT?c2;dORY?AK>a^4AoF0o+-;}AxB?;jPRB~Hxx<)=8~JwcxIupn*PXfl91 zi8m6)5IPgj5Xi_i>X`Sk&-Jb4G)qOJg4*g7rQHz;&&XVl3Gt_pU%4i$#4cY4GAGJ+ zko`fks3|n_9P?A!ZP;F*<_wWE@6g|TTOx?m1<$QbnqErqoA<0)(Kn$rQGcU3<2n9$B3>E2;K&E2%pn#4RLefUiN_E=!$n_T~17vv8}WQ2y> zwSZ)zR@zzBP8UFDQW(mbeC?4Tx1I&aS33!-%_91gk|qPjV7+?KlGeXC8`NWlb&GxS z6K1J_>m_2>5!~$m%iG6&H`o8zq^rC&;hdmp9ihhwBEh0Jp=LhUgU@=-?KZH(YnfBM zFR6|^rv?UaHmbxcf;>7}rX0F0$BBUM!OR8e}e=AJcszeU)SI6EDyjkhnulu3CXHg9v26fP>mVZtzVIcUT$1k$-#EtBIO{mpb~)Q1Vo`ERA%;^eWf zUGu-+Dypauw1UNvL-{#dx{B$urpJq0@wb^znqnSm@eSTTw=OEn7sJiCa9c3CyK*TB!DrxCvsXfj}z$5vta=8uBSApL{5MBG}IZ4-+nWye1!fW`j3-gcDC z6IEWFtot_#a&x4|(EI23+IfDcGHqy=1{ZE4Y6`N70W=ktcH zjVFMLQGi)F^LcUQ^he;WdM(cSHUPBacARamFYZ@bTS3e8XxWXwX)*7S^u>@8<~0S8 zVk&Uo!?@roX#LzN{O9y|)S!C(Q${uehZCbPHAQ~mL%rbL9 z%{&Cp*8PE3uE9OQ-Ai=M$&#f#5@`0stE;znfWX?r%9yLwV<~7YIoh5jG`EL4Jr?o! ziy%{@6P58$+$F%Dmi|KXCB-rP>UWE>G82+Z?qorx#|RI?ur6LH#7%*1;3f9@7lpY( z0z{|XCRvevYcFtrnZ1x(l$wA{Ue-#gyM1pEMxuqi{}cVqNbsi5%(#Ca63|_j^$h7k z`xYhJDoyUJVgn&$fTQ)i)x!Ic%uJ3EsUkR?DacKQyl7fjI<6Du$(5op)wEu)#_V5| zM#4&x{Feloz6vy)W#PY1K(S`fRS>l%)E{rf=u^RfmN<~&bY^G_mF8MX_;=o!kkx5C zCdk#xZZ16&)D??-ugHjhR? zm{l~!D6lZ<;oC;3b@Hb#s&t`T6W(1PTiGt54+Y*&$w z$VAMh6$L;h2X;?;RMOPEESw?MBETt|kHq8}5N%0sy9{vw8T$z_A2iU-RtO6Ba5s>p z5-Fassq58U0@PfjhhI!5X=K^piE`Hq=2prYn6oK&Aq~SGqps>$t2Ck5F_d|ksD-eT z;Jm=ps+}^MZI4@&cYLbw*fS=YF{cmAq-@C2u0OsZ#GJ*ZJ{(&s%?ROmjtgIzozmQY z6Rp3fl(Eo0h6?a;!P%O1oO|lr<1HVbBG_zxj67pBini<66!iwAvsUUWg~%}5i$}$ z|`JzO^Zae8N3Ac>dzU-Bu;5-thKd*JM#~@iXfjnN_mEr!lPHC;+2vx0P{(JhyQ)A78 z`P(d&EYed%0Aa39FPPLhQCu!)oE^~^%9qw4!Wqs~+-qH-EN(bUXBtR4c(qXXTldz3 ztZm}tR|Jo<2Mvyu4gqjGTJ;#7EN6!f1@lL7@6+l*$^OHOk_4qf&vvH(m!zqFl6uM! zU{Gg(_hDk%ktE!mR%WRDcy<)zw!(KDpE-_HW4)t5Ez_j9n~rzO@-j<5xFN%MGI_;K zpufvq7t@?zFIy_3yVX@`w)e-j0-OnBcbjS3V{8JEB?0xxG#FlLu=oP#{eT!h;};8aF-4V=Y^_g~qPCfpS6Ym+Oc9-m5JZVXSfb;MOF?QRVBSG8goazWj*=s_~1 z)$+^CDvr1f4uEA@>*;Xt7sTY&nM`8-2_#(=3slEYLrUybP6RZWnDPS(0}N+Q6Y7RE zFvypM`Rjt*TjJ&tSde9O{*%EVRx5HzC6-Z&&oPpSIWg-LlKxz-1MYrXU}9V}hsT^s z0@|lL0>1h?rKtdWZmFBU_#=|0WkrU=$l*yOnl9WeX;siy@!XWt3QT8eS=ktK*U~vV zy@FxX&BA$=lZ~YPRNP<^eQJ^i~D9xpB1kw0dZ{V5+5k(&-7HA zc&An43#S8#brtO{HdV@u-BTN_#c!3SGQGnRS1XZQ)P>=N-E{sb)SYW*4x!Hb8P~TnPC13cH{)$O$I>Mo&{=JEA*|; z8xz4MHs~x4jYy!^E2vBS{{ejKB+{fG#^5*Qi46)2_g8%_w~K`h_o&g> z@MmD2cQ~kEbUIf+9#y?9(;gQIfm*$?AG+*p=VyQQW%d&SEG?z4>*V5nz5N`1VU!3- zpvPdkb8;X(02Dv@IluZlQ|HWSA#UH4b6qv(I+zywvqJ(Vn)bygqptSzfgP(b`V7@I+Qv0o$pdD1Z`Uyf6tdcjKcU z33rvu3C>(>=^vL^AJ*;=95=q}it;Q8Y@ChiJV_FZ3U{NY#STii(VDzmo^jN>M4&s$ zGafqS%y&b9bdwKF$(MwX3&09fDF^<&)(8abd?hxotOr4%+_J)wmF+6HR z$W2ThZYzprQwHYwl)n`qx1|`9X|+k9OS68UhT|K1DkH5UB(Su+9c4N7I@$Th(E?3* zu_QAf+U@`{t$J$BdR(it$^621!ur(R)f)r)+Z}I*-DJcP zpOzTPM{H#1{iF9470cR36@Pv|5cFvbX=RAn|L0`Vq4&qE2b^S2T65}@*g{Dyac%4= z@ja#TvH^pk3yMnwnM^FCcoe=Mz+IDTLp<~#X+EcE!_<0LN#`Fz^EwojhX7nBl0cfu zH9>hO>fJnr`*2Z)V0_4D-xL6eM=Dsg9ifB?g&nGVJS&fYjIXkK9x9p5b_;brofJ5= ztp6y8Kb^EbtTs~xnQK{Jb#upIcc=uBsBz;YNLvgG^hfm64}nGHotRL%mz=zGg*sAx zBk9}t#t#M2dPI!jI7x}j!6{fc8Gch@dizjDIuo%5f#y-sRpZGrA1YRzRIBltFf$y6 zC2*)V4j$8f3^;Rhy%(>;0ReeILy_v>tuH-MGzO;$DMqVt%#&b@A2h=%KK}9)Nf@F{ zy@`jEG6j(fG$CTn--|j>{zy~O9xn^?*X6I?6K@Id*KP17#$5RyfF7VeY>GK|uT>(f z3({C`BE+mw)wSZ>GRLTcM;s&s>LM=;#}AayT7XWPHRSWMbdStpSNJDCd(VM*rJ8Cx z;cm$Q+9=k_n@fb-CaI8?e}+6BI3f#}#ddtIz}SpAUXZUMEFb5336RZO1?f1~Ej(?= z)7g-lO66I=h;?#T`x8NKgyN-eEMN0^5U(sPv8xg;p;s7nFnLyx4-US|9T(3((+_-0 z_jUC+<^@s^g(==cSo7syB$=0Q?hl}gM5H@}<`F_KR9~XG>LBiZ-#WiAak301pvj%Ex){Dbm0&ur6mtu{HH-+RZ+`ywx5#XwI z;2~lnj(;x~nWTDd#%U!$m<9xlpOyfj3At`s;B|^*nqwfP2Y!w#p%hDoihnG~ALHM3HJFB`_Wi5CYy`~-;6h>JlK`2pHSQ5YSV!&S`2#(;X!{K;b2~V)jN_V`Xq(8z=ftj1c)wSOT%>2-k{eN49xn3l0xy8nA z9{~B&Qbot5N>gtuYi~6S3WHYwQ(!P>qgHAF)a~2h;NZt<5{6@y)>fC?|4p-JK37# z3r7Cc2^gwPV$XNQJHoWE%_0cmil32i1K77&wdV>j{UpK3;I$Eb0_iz-uS8rt_`gM# z6eV0%UvZs4Xl1p0{8>rY+mNqE+&uptsR}}IA=~^zDcXR}6w~(wNSN*LzLN#X)k65* z2tkyparzfY<=LhKBqylEWoHY|T!vdx-LHwBBh-{pR8sQCxq?iY6v0P(^!dWPx~7X< z$STggu!K9S*zhuPx&-{4;5@BhXAJ&f-QNrJ_s6Z{5duG75EQ|22KK4l3-LEZfzvhT z9#GP?B%g+gR>9td$MesKm-M%LDyI)rY`(63UAQU4vN(&lT5l`@Ld~d_WMTQXlI|h5 zT*yWoQTi7JgT`3<6Zc@Tq6Xd@hDl>LAuhrLOcr`?7sdo&l~8Q7=v>IS!N7lGtbm=q3qyf-eu&BbS+);}-y72=QK(agmk4LC_yW?_ZqLH2vIEXbjQ z*Z<-(A>L(KQ5AkEdpBDJtThig?Ga)byFE>+guO&(Oi(K)^O1>E@dB?hSk_p zs6WKUi0Lb;kmJhY@-T!`I8>b?(2YdR!zsEqmYNI%rDhX(YgPU2pRsvPO1-uc5_fU0 z9zPNUsavZkUE1RbCH?Vk-i1g3c{g7RjCD_OCZk7*Ohw&vvZ@m9yTDLlXfh7QaqEJ) zZ&)36N@)vSS6L6#qXh-(DBG(-a&LK*oaL?=^MtwO$;b#%Cqd4J8XF}I<_`%o8)0w6 zN&CDFC}*Hw#u~BDh5#_8A{I&>IYVh?C{j1u;6wRWy)mZ7J*u>e*Rg3Ij18v%xE&aq zCR(gJ6-ax13(u#$tAq=*bxG-X*0cg3>{P1YI?e`qxiRnnj4@=Zp+6`Cl1JIo$Q;nk@z^LlcyBKTX-w-XR5Q@VU;3) z>eBfY3G1mO;wgaOTEJ4lmV#oFZOvV%s#Pi@Wp{^StNGg|SOJqo52q8PRCy&U% zF`MV|b|5vr^|K%mCeISgL;9guF)%FExh8>bTD4!NAqez{GP)(5b+oz63cm7aSzMlZ z$rU%A5NP6X{!Vf0*WIxw&3q;tOHHvHTZCqIV9?kgB_74$g6SRsabWW+zLVyVzDzCp9wPE(uFe5RL>B# z3NuMK@vwI2g83|gE{eKMK4SmM2-K0wD=c%tw~NY{ja692l$(!&`TK3VfSAdo>KSsbl|8;!1cfb%@JJ9EK2T0E1^IzALv3i5|wBQCCD z>YgC}5c>u)>U04v#e&{cl%!?%D)Mxun8m*qKs9pcMHS3J>G_PHGU8H8b_p-n+~1?5v0noZ$Wh8RCGly2&)q67oyVpS8}d_df^SR zud62)*Kr&i5s!8d=0RqiXCg?hk)y#s^@FD#`gI)@H03wiuRys~PbSq2)(?bwq3Wtv z=0)>B(rz=7rHqZ^3*RdsnfzB#M14(=nL$yh(8Kb9H7bCnHg0WPVg8?h%rZkwoc8qw z-S=AoJ%=F*CaV_P6YLF-!+yHh?vV4>5z6jC9n<@tZE=(kbHltEk`^LfoG#39Pt$=U zxx>JsaQtR%{Nz><%ZkS8fGm zqB&q8;5>ouGwLNmLvlazVTo}D9!&?LxjjW;MqU?}nq2~EOA0QMPU0W+f-plxX_~fp zKS_E!gLC7>`rBXPgvuoNS>c+40Ngvgtq8gm`wxP+)huP)&&gaMfq75Y z&Wt*nm9e6%?sJ7_m-QT9yP)yzDIzOfKDo@+&(o8N&G!dEc}uD4eN>sj#p^=N!Ca(Z zan~s7MelA8jmXfxk!4A{Nqq=FUi|3F>!_D&uBQ!e*V z+rco(zyC!!sBjER8|?0?(Rt+%iV?6*;d3W^XAr+phWE%4V*avU_r^8{7p-B9VQcP> zy9JpHY)^^-_J>12@~B{y!kX3=R}1qvBb#&3kgzcIP(WHTaq2bW)WblbqGBN+i^Tmo z%@Wq5P!mfX&hI9aw5-x2%j&XU@Hj8jdy-4wCBi&@C^{H{+rAIL_COSN?pco@;mv-m z*%`+u>17ccH0?q8G_d`V{NN8VC{=CV5IGX;k?CSFS16JZ<#ZclO2FMue;g^y*_c*I zU_POQxWh?yNT-w#(}xADH!c)p!Zx^9gSGD!V*cu?jNxH?R8ciW`lSolLJ)nbV13O2 zY&RuM_Q84+L@V1q0>^a=@LHR8d8$7{ILtkJ>g0%{`P(0_H+YJUaTemfb)=&bB~JSx z7+>Rw7+w0Fr=-cPAkFQu!Lb048>RB>Sm-zsCWO{QXc0|8>G$a~;t?fWdQ3Ek_XRK@ zl5x;yNsWTg!kpOZ_@btPD%*a2D7F@4rq)_JIHWx-wh`(o`*?L38#e)TB6UDg5F6=N zPA~?4rHg7x#<9Z2d!l>llLeR_xQ|yD*B_q<;_f8N5k7Z%33XsCq)W^51kuB`v1U)~ zdQwqqD|4Dt7wIaTA`oo%4kTWH-(Q3Re1U^7mN!@(<)FKC4F|kr( z&S@YfM~#gj7 zXKrx2;v+$DUoS@|y0Yu(MJ0r>RaoGko>3s^R+t~L*iS(c!%dkHt0-Y2sriH4>7mcM z<;()L`yI>kKwNiLfif;oJ?Z+l{~4%ztJ@pvxf5t&Rp%krjVJ{CP$ZZ^L-b?GLhw^r zXTyC0`QJ1>K(I;CO2kQAKx#s~F3J9zt!p%e>5pOk;Ne6lMF45Um>@TZ_gs1NS)0RE zxk?!WvaWHd|GE(Od<-kp{I=7gXBUOJi0GHWVtWh;G&`_urSL~_Uy12rCq-s><6QE& z-AHH?6K(d&L;Stk;HBvF_H~a-ZFm9tLfNArQ=a(h91z@sWA%ww49bDll>uM}_ z0f4)uU}&b)h63F(-7%(TD|Qp+6>@a6R*ONU%|AFfwH{mim=N$1Y1e{;Lr5o2+d@v4yQM?<+VvDc{y2Oh)6Fx=ObWD(A~LQL1mEEK zuH(hy6Uy-J3YwX?%pXz=J-JD@Tu$2k?_@GmbOPI7Snej3qLyCm_aNpG4##MuzVQcu zye^<%=3O%hQ+-0m3CH&lrnMdQ`iD$-!ig)EPs-~L27h@Q3%LPk!@N@iMM z{3sb^hwFj7YVwe`4LcZvc{c!?@36%qmaI1RBLq4xW~&Y_g82MDu+&hn20L|vT?L1R z^vq<=>lUC3mFWkug{jqK>%d4zG9uqPscZlWndBAFh_lPAjUF|sO;(wYmRXsTxW9=l zSGlq105KdI1WbY#P7ugg@cJR4N#TSOZvyZ*>q-sEUr-_)+#q6Ov*4eKJlqP;oYNA= z3o(Ne?F$C~_MZXVRZRJ0BgOi)r$Bd=NI_hu6Ndg~Ack>(S7T%7FU9Y~H-V}9H2!4$Rb4c&je+D$80Gn6wfZRDLfc8$FEC>!IZ~STr9{f zW<|~qo2+pQp!!x*XiQux&Y8fvG@t38QtH;8(6@>;W^F*6Nh45t}XxHP%xBF*=1P%a_OwScU4bqFi zKWsXGbCz}EeDNUN)18t0Gfzod_4Xn;-n~N`s499%+chgn4dN+Pf8EQl{X4x(8HyDTnTd^*1a> zxcRm*OmIp0bFGL*uDJRrK zb~~)hahL`;o>%(E#E3Jte2P< zgn;^3n8{RSVS(?z{1Av)hpmVx`ML;h73eQe8^q$SQIJbwDvXvRt`cId?BrgvD^`CP z0Q#U)>5VOwaEnG2fNHKy5^mzo5R6JK7q^lGNEnmkfhNieU(mI7TX_dP?|;(BYL+O|;| z!vs&iKvI(gp^z)UxcEiVCL=LWhmcTW@=HZMbA3i>w|6`mp9}6Dq{oq!N1w$sItp(-VSd4|aRKK%tWySL8`E0cDx4 zp@Wc!`#&f_Oj#b`=!A~Xok1)cdH6?_x;L`ELLIV~5cfv*PTZGNV_#t=Gg49AdW!=y zD02>!U>>_G%KB1x4<6}$gR)+!M(f?NypsMP-c%fv;+sO;D&2re`q-+32Dp`wLBYyz z7P$!ANYmS5?-C;FE30Q`{Prz!u?mmyrH}M?<`8PHRIb#d0Rp2i&`6T zDBX@8`JbXb83Ckw5H&yHCIyeoHWd2s7-cmhbr9DwF#`hB34-&wfKDU1Z^y|3={;79 zaj*U!=ONDR*weC^{-02Hmo<&+UaYU)73Pm}^5M>*KjwQENZUyZ5Bb!UfE+0Gb^pg$ z94f>maBxd+4lMLu39*eaD0IcV<;Uch)tf4f7Bx44ex zF8VQ`+A)h9mngxE)l-LEu`(K*rJ|#AK~5+ zJeC&3@1wQup4eUqx}%k~+Fgl#l!h1s0|RlP{>~1xHX9wpQC_zMi+si}=1p!A_1f;o zLQG~%XYdpm_`-L_JCKYukR4tsA(3eimxlWmAeZ?=@)uG5bi4VEP@eG&QAyN!s9rzF1(oU0~DejIRsxr}2lxP9(+6-%;o|MbT8t zqb2@Yf^uHq6vPqxiiKQ0DZuHEWxu|KL20Em(|t zQI~v;{R&8jXM+8b(o*d~-;<1>b zY`GmgNN7*w1Ues1Z;8;GDxXM6AwU_R9T1yMVXkKg!4(dj2ua>WIE_Y`%f4$}@hd@QIb19oNRzsU|4gX6!!lg2 z4v#Ry-UvVs0zqp>V%6nHGih79Q9P`*d{GY~&A5&8ZbE%Q%aIM+3I!at*8BXg@{?yL zfm#x6>`P`W>4w9XRbLd-0! z4;Kru#8Abo?VPWXgj0H|IDo`^N|~S|sHYln+iC!=vadoYDQJAf>OhQZI`C-a)E01q zKo>*@j*G<20&)KX z`s7>$p)Ot@)C1M2Ye#Hw(*4RM!p%bUF^N-m2ypT8_#V#-bwhf(nkXmY+Z%wm?Rr~# zNO81x6Xvpg$cBi4a>Lp~Xy#@-b7iHxpCEr#@fx90nTK-)lOL0g3@PX+i?Z0_PC;&k zR8cq_KUkJ)t#M4oV&|PQ6UPQ!bj;q6Vs5IQM&%q%&~ag|3Q11k+8-*B6- z!;|I>&1kP6CN76C6hSAJ8758hJ>s!bif=)Cy=>GLhff7#%n1C=!qtekm4+q8^eWc6 z)BGbF3wq{xF>N~OJo`|>rKiSY#TkI^0hu5O+Od^{nTD9^L{80%i>4o)&g_&%+fnxD`RtU`)K;e;= z-c?%wEa>STPsehd)mRiZnZ++|40rEXfTb$87YcRQt`LzcaqMi;d1Md?Fc8^i1IHtosN^0WWSU^FGWmI4n*v`@M?nn861{7I(TAn^P<9Z zc?j!nD99Be5pr3m;rm7jrVEnfO?O*aShj8QvHXW1(+?R@RsyKWgwa{7t@YeK~RO2aJ};$kcD4H9`wuu0`po$M_rHyrIVeLqsvJ`ykd4X2jhZm ziws07l9m4a+Z70g!GtD=C4^|jojrX{RF&{7K@i6D_-O&mU{pp37;h+L!eB*?C1qmN zIXi&5W!>_2$ux?M^Tzc{uyzMH_z z8!U5RfC8ed*ooBc6Y0E9zL#bsEJQmzO$M3il*aWN{` z{o-m|uJ!deQ<$5<70aj=jcA7l48Fa9YUOGpaL^+COSQ5g0GR_Ww!jL5HrIh z9?{usch{mfeRYfao|rSp;?Ef|`&W~}mEK(kSLTuW~WVErtVx@YAWDVr#r34xg` zn*Z2G5be`ripUvFC0#jtR}GhW7P@Q8VjSdJWtwtpE~XbW$?>d`rh2+?lN#}ay}%eQ z>56OdoKo&4?uCSv*t;l!;h)yTfw)ANJ3-`XRNrYXY_Jcg`GD{o%;lDS0T?@U#VL+H zF92jjRF#bx4^);2Rdf7_rT23wsWfdv*pd-S$-#(q{stDv1VY{`B^X^6vi|O$f8e8%q@^|(!a#{o5BkCUED~~t zNY{%AHf0sGQbEPzL}j%orI$!$yMW5H9_I>oUUdDa(+ER*k+96xiao)Qzga@OY^BX2 zJ}FBof=ue<*S<$duVS;~7fN^#*!ytjS*i_y(M9xvxEr)PFTr@G#NJ9|F4G+bLjSoP z&Q0Kb8gOe)wc}dWjD0%!%@pP`Nq6qts{Tiy>%j6)i(RalAGx$jwG> zKwG0r*+&GL%iAk>&_9;bNY;r?|54fvL$!c&>BT(-(9IUzjQRTjXlZGn5pN5E-?Ivx zCE=ca5ZcbAN;6(o!rV{@CtA&`1K@5TD@0E>ZRjy*LbTx}sSUTI%%_tg7O@P<1P~d3 zg~yfCq_OU9xDhYNMvuz_Knf$8G+ zWmfJSdfQvmkl%khBE!>3DYiN3XG(l!QmQI!#IYMOEIf+21 z=)3WE6?RTN)|ag4VDeYXIy-CjI$F9(PM;GitsIfmn`FAVx_%g62T)#Ug!dL^K^I%U zsysflxpUy?)GzR%m{pNmRv05~dN+m%yvOa;VEj#>hm%EQgeMLTE&yDEdOPB~0#Ki> zzWVC%>qGe4tj+Wrju(_R$ysydoO0Kp&OtB@Ss_K+VWhPIu;kX__3x9)WI;D6tI=;B z0Yn2DO#%x@vG!d7Ccegp(e<$_;r@2B4|=x#Z@x5+`r={2O(a=1;HWbkX9#mU8%=J4 zaC6KEM`lh{x~e|qNb;CGJnz&?XiQpnrodboO-T~c@kbR{a*)AxFjhGlj9ze4M#Nf{ z=Pfhv96r7nu`@#jjFW0a4Sh|BJA}RWKy3YkB2BEYTHL2Jq~jV5L2u5nq|CO+K5h6( zN|N72s2k1o+YlBi$m)+CS7b$X0Q05DcN0)fDcUhOM>wx5NG3qZnEDA-;9l!yg4l9x z@nng$VZWV(>}P~|I63K|;cUj+C4>=8IxjDCe346r7jo6SQji9nz?KN2T`_h7Kpr5h zxrirurVzGQH02#gzSR?50(uT$^E(uW3vdZ^lX9=9o>XM%#@e|n?mW53(owBSyT08i zMHWo2Ik_KmY60+?8kgmd{-`L^nv7&)$$AIL_>&AKZIbMHUcIFCmVc zm>l+C@9JEb^XU+(yZo6y2J)P19qvI{bgcjng6{kXC(m|uhd>X4?B6ha>4`VXj5;fk z5k5NhbTUE-g%Os`d}rJt#P#VeL)JrteHEBb#JG$kxr+4=ChGhNzgRpS8*Yu`lytuk ziFkyGhsp$*`BGSKA5a!O3)G*Xb-E*lD$oC{C=K(q*RUy0kXJsr0XG!T2|a9rU7UD4 zRH-A$3o7njbN`|1?5Z(qfr&SZlTeo=qhv~uvNK0G3W_}Cx2s%T$%twKG+0AF!S&X^ zl`GFgtio}1S@LWU&&1BkXiwa)v|HAP*;hI>$MXU`Y8Y~HW53}!0H%{{=yl7pry#c; zyL825ti+n<7L`#IGeR*D*ZW10vB5ftYN-+RUltf9Rhnc3;uB`VQy4*0R)pP%8s%_LAE8^H{uJI z7C5gu6E;3wBUX)@Z_qxAm`NI*U)c?Fe3pZBT5%KXHo&zg&!qiWPV8@7%JO{mGA|8CXUAmOx~Fh<%GS}5*h_!=6X}`8 zj!JZu5KaX6iZv$P2l#uj{dcbI(%P7M1MQM*s zU0kB1sjPr9LtQAiv?yFF+$~~^kocQ%T?ujL!@UB29+wxnn7*wlxY6%`+(?YC(PqWl zLg?sB;@n?!Me*xE+S238zXvhR$t>NA2)VmZ6LQkbd=&k;FfTll2q^N7l9}~%bRCMg zvZ#uKJPRYfDD_xdsHbap9pSg*D$?#*b&RVY)Tn0)avOL$Q2I_CC=%*6pn#W&Q^S}G zb8$j^foA-ytgi!KEX}xc1-WTFFGq~_IA6#(yFtC=HF#lJl7n@MAN6ZN=2qVqh`XC{ zS(%ejVspUuuB?&A0eAzILw0;x=0ow0y)=e@*A#8(uGHj{-Xp|I6LyRYY*T z2>aq5L0Q_UR9a*0Ye};$B;h}J8*t`9OfdaK*FysH^2mCH z(*{;)yssGdRrkJZp_n%Sd#1f7w0QM>-0%U|ELx22hq@RL}Y9*J9Q4WJ&bA*i{pV3L3WXY z)_Wz=ULzn!HP(25Qh7c!kj@dE+hVf7+?Xz|ve-jsmynE75C>wl2gxN~LEx;8#yd*7 z{~TM9@wr)nTwi+(czhaiOl5`Pm@*NSF3^)Rp&kPK-*MY_K9j1O=m~h7G$&$xZRz5% ze4|Aq=<}Q{sI=Wx~cv+5! z%L?LfmmqhGhvJ8-DQ;rScmgy}T12i^?Gi1*GGFl^EBu}ia}dH6Lur2?U$PSTQD9PiWv>}eS%yyF7h-9=DlB-TZ*ckb11l+q8};qiWwyd#q+{o2hRAi1l{Nvl5WRn zKaXW>qcp8V6s|Ype5L#`PFcJJK&2}_3uZ!bH&2YWc=S0Cw-Unu4jqJ?ejbRgRQRFNuBfNhj($N$`tQF1NXti)VOkvf7D+df z#eywo`+pUcXupj^mCn3G_>?3_w7gSbc$-R0hQEaQV|p9Kp|EF_pxg<#d0{jYmw9@TUR>pM&*Xi+~5LU2* z^rqB(|0V$gP%YG$?>$mb!>W=v@O_eQ2<9-%^H}Z!5NL||H!7ZHtfMqJ}rO^drD@W$d_ENkU) zA_nO%LLf}Lj6lD;kP}l|=1h$%l`sv{t#N!l5l|$K&P9qG$RylKKPN!TWFOHW;&Y{# zC8%;m;e8h+nWs9k1ZQxnn74Qd@_4haG!y8?x8om>Oi0G_df2K1+~*H)nKGgLP^H>4 ziXlfxD8!Sj&*JxA0&{IS9_{H% z7R7q%?YO$0E5OB&+A7Ffl{jvxqL^YnFfYfJaWMjUYK^SL^2-8vMANAh1HvSje@pl{ zrD{YB=3gYxT`2UEXkjMg<(N%xxeGU#8CF+0+1TVe$L z??=ev3{bX+aK3c0iB!6mz)WUj79O)Zutapt)Y})Amz5z=V57*AwQ3aQcpkbPd1PlH z9*5+i62BIZR|x`zNZ+#Paz!bQh&sTXAjGBcxo5tfEx;r~1}2cGBq@A!mIr2iX{~hi zckyo5NlLrZysRoimPWkx)uJee8VpRCnhlPEkOsf+gB_KbFSxV3MwwRa#@($ zYq{r04fT~(fZd5I_msM{N{~Ozs} zOf6y^Lh1fkV0AENwj9ejGak7{QAdT4zVXPKqR63-o}i?M3Svn+b*2#4&|MuQ z_Q$(93-*07&tXmaFxN4_g*7vD+Se(O*^ZlncCm%|4X1hN=Q_6`&VyJ*fP)Z4NF?Z{uTMOWVr7gIpkFVM(84JCWn)5zmB5|wWT#7 zNb}nmMmQ};Z=ppE|GPL!PAlf*_|q>9j51aH(?F&fMs`-*}{$afmRWO11x2t&3 zK*v77e4lkSZ^qVPZHpaN;gH)eg&RvUkZ1{Ktq`L>t1oO?Vxs=n7?Zx1{<+*=+b>BkcIi}3AhrELDin*%>bGBv5QJ7l8gcw%*Wibo7ir?+ zm+H;vRjM<&zlpwTAR6n72BZ?;v=S}qBLUC=Yx$*q*g#&abV%PiI<-)Um5vFQuA0g7 zzqk&}Xv4c*SeVUFQ6t=V8`%m`-G;vDf-MUAZXzL*Glf~vWg>?Onx7YY<0>)M>X4}a zB+U9zZ2{1Q4vdfgjRSKbEqah_|AWD)G-$VIyiszFxE{^3;?hdOWUVk6bv<(FqE>8 z;a`ozvYAquMuH#t6FjwaTT>Ac+(-Z`uS`p5FU$FYZ2Suoz@QCR#rD<*%MJQl1P}F% zBL$Kyxj-(=^?OFyyEG9h!Y}UN-AXUp+|=_yAr=jTD93&k&3^={wpk~XTKLL4Q^Dw! zHEZ2G8U%KxR=7gp{JVK0-(g1(96}?w@~1#np_&_TrAF3h51ajHBgtpD5^ zgPJkrvIDrn=L%Hur&gC@ip}r8Clyk$A<{88|4SeZB&>av`HV5VTE6H7#!9&)td!O4kelSc5MU`VaG`4OY55>g z8R&i4Mp5Yp2#cyvT6kuPSf-9W!okRl=$)P$i^r%t(P>FL$vL1Pl?bB&joJ?rViD>d zpQ5mMpH~RT$iz-0w%qB(VnP{%C={kEe-cXmTP%zEM~VO}|FZLmz}2$^#7bR+ail8N zar+9gB5^(s(S@oCnB^d}o=MrXx2s~0O0={L_U#{IjK)EG?5`1KnM2fE4r*n}W$O(wpHY!Q=)g6p z=_16^LKs7SSzHNV%3w*z_vF*0*s^P}Lco5C|X4Kefai zA~G3S?T5?1(NCR_Rz3WP7(?>}XS2y^q%g}kLKW&-KS~IVmCj1)syp>+xn?sz#C;|P z*w&1`;Wq-T_AvZesq2QGnhX>%pH8mol%n4vkcbiNDzm5hkyBEBka5Z?tLd8kHv;h^ zgNiWr&DFd{td4lhF5DhJ1EE|g0|xNLML!3#EEa3Ug`Hjia3zzEQlB}^UO2>E=s(ko zaRJL-OwmIhe$4}*My!%tl2Nk@Sk!8n3S(t6j0yoJE$z$-TI~S}Yo#Y3J02_iWIm7$a!<@EBh5ERQNX^=FMV56yWfH z|42y0sT^v|_R}8$fK$gpec$O(ULxky7E=USx{W*Lf?6HfO0BwbcJByosUvM+fo_p! zBRMw#a*Z==7JM34;fD?22s4`f->DdkBy8;smVh3LA+|7-GTiuQ1X-4`!3KB#N`RHB z3QJtj`=!_m8%iP44gpccGlvW##hw`B=}oX zK(IgyO?Orwho%9BlL?x7J}B6-R#1jwpu*1UPKDB$E{ zk$+5xHGsm2EZ+HF3NU`4g`skgDId80L>zP~({?hL@jO1Q9PD5XL(Tny;$~Z38uVdn zgj$xdLZc%iRZCMyF2~GJDs5g4V)-jcs)b%i%DhL>I%XnpB%Ku8`~C)yl#iXL_t4_yjcK zSTqjY>)bj{_LQ*20p(@zX4J<_%Otk(uu$77tQNd5;Da@~^~xI9LFQQGLBF%%XyVXlk&z*a&1bfyHU81o$E$ zWB|Q^nXS5bO)8Xn*iw$1Kf@zO3k-r54#*w8B*RQIdZ z0XoQ)AI30iKFI&3pgkg(4nW(COZ?|?bX;~4ltZgGh`wQ&4mBgIG&}ibg_eS`6uCvp;=c>FZV>j)fjy>LHyA#ce|8G#csY8; zO7kCe-#TL@vt<8N9G1~auAM6KX}bW7BK@?bsGZ)8B0228VP`^5Ay%o>3L{V+Q+HVP zhp|u0eQ68xhzLnpk??DU7^wgQsV2MvWVvEK9*(H{3A@3_RblB^NB}**`)Vqj7QwYz z*h^5vhs;&z(sks#f9#QE!yv1@Cx{-sq6mfjy?ZQXDtuhN8qX^NEcv1`bfcR6f9v&B z)KxiH1VV0v+yOP8RoMDQmyy>RW8sy~1MEVDCO!5SbA&)I#@AT1Vg^X<`N47gNVIff zp~o2#dqpG9T_@UWbCW;`YQ;qi?0fZQ46!F7*?w99od#PSSR9hdpX1Q}Y_B244iPMjIx- zy0$^9A_28p;6}l0?o=sk?NdwhVP6wQN|jVg8`0{OM*XJ+T0t@YPfzHV2(bvmL`6Es zWx^~=Y%Pk-JiZ{@axc52itsNnREeQX9Z~wPAdPownGmAQpHrc-brO{E=71o)Ol7@D z*V3H-KJS*-aHlUlMvs$7nWc*=)hMl)SulRw2jEX3$g z!mYk+=k`klS=x;AEz&^IVNfd<(a|%AYa0YcJivI#X&is~M?l6rC@WF=-w*?nP^NRc z*2nm;OvnT_i%*MUgiAZHZQ$P*LM&k?za;4Ie3D9u#;FY>>`?;>l@kPfKlq5flMzN6 zmRF*8Sdng?J*c1ZodI+L7fa00@0N_CDyJhI|5qR*E(^Hoe)pZ@U>Zwm=>S=gB=w;TJy)KLT=^Ow!c3~WK&uiuS9T6jC1YE~hOvK{4c`(&IuTuo zK6}RJmIsL;RV+3C_Y2-F!C(oi+TU?3g%n8Ssqps;u>wtE=P)%Ejs9i&0C8#kTJORT zUjz`l?tQBAo4+(48eGHfj;sC-KtHI7sF}yonynx64`3sOMn;M6WrCuD!-l44sTOD~ zrF*o)dPs#|5qm{JN@u~8(Z3?l8Z5JCyK7Iq#U9}%c3ds7Bm1AcThEM+lW{@Tx$^u2 zUjrLqFmk~WR(~=^&~9nke*6S}j74<}u*^4plM19`o%H5!35gmMd&`g>O8o=>0;7qd z3Xs*+ReH6uvt5XNB@_HDCxI+a#bs4_j5k5~y86GsMv{tr?%B{53?D!X&Y4Rj5sn|o zpHYeaD_F9oL~Dea$D;D;3Cc-;tN^eKD>gf0KUy?b>-bF!Fgpgvi z((jM`p?0g(D)O)09WC@>4H=E*zajwJF&L01Pgf-H0Vz1oRa(Z5I=xzc(Egy3cqk2k zzO8TAJpN!hZ$?apDD=NHqSG_LQk=sMOsq)?iiUa}58?PlST;(uOL;|oSSZUL0kV>? zek~yminaoo{jx=jZE^;}5A11T8=;szP0>aDqWVf1O-PTEkcdW{{+!rbxPeSPLBJ?B z?WB~h?lyU88SXg!iOH_8Q{uP6jR(ak6;u5iQT!D_SfKhV{a!&<*^KUGL05(O&{IH- zEF5g0Bg^$8W5}9gzW1l%i+rGp!SbNLO|KeBw4{d(Jo~7V+i>#84!l}AG+t!JaOK*= zok5Y*`droTO8>=aDK9x?*!4$WPIdhBbkK-frXYecIQ)Mgxkg`%5hC*$b!)-o0ZP;m z+6q!Y|EkX7{y4(GH)z5!|02+ubv9?de3{-YE_SIR#v{S57G`nj=#Tb3{W%IIAy=&J z&ah~h@MB1%(zn-plzdPdU_2-C!qIQKg+jkc?=Y>M31q-L=S%=AuvAHC=Hc|gX8}TL zNWx^&->+ArCL)0}LK2VLpKVd8cV-Qk!%yd=yfNM*2?o{Rw(~(-Mi`_Amgd6!8~$Z| zh-#e@Pt70ix5tnUf@m2qKoM!!po!!bVU}llUq(~9690?d5n3k=NN1km{DZ}SGprGcZLZGR!J)~T|aLLy~2v(mjM zuoaSxLMSoWW#0-5=bY8f0v7cu6hR8LAToVB4xhtTzHsL9dohNw4K3BcpslcKO~VlX z=L)kD(( zz7R63HXuD)Sf4p55Mr}8CrlgYuKQdGMA_NC!8HOtTd$Vjg?+-!PW63Lh)lmB1YUd< z2%N}O=A3&jxtcdP9b1-kfO&qo-i?=}2G1^uZvFX2xHW9uw?>X6F>qgI_+Y$?U|-2) zHri)zun|}5dvwdituc6vbEp*Hq~ayV=+>A3>Ks#gKai+jH?z-eEAN0J_2Vf*+B#WQLb0KU0(zp0R}+e>p}KuxMD) zfvdoQKJg=bT5y)}B5PXb2((OcuUC<_wswml-WA;6D##+F_d;Sr^PlIR78*5KwxhDF zKIE2EY|IM7WgX86vOcMZ**{wA+W;VT{uEh0%lDPHn+J|cgniKdR{~m!Sz0zF7~v}# z2}oL6=`~v!#X|6=6@Bg-vtA^CMh(sRb?2uK7#us1X-}p9nSxoPWhaZyrdY@)q69Q( zmxN8SPYW^9a62Rlj9h=pC?JcT^AMq7`X0i}+t}>tpys(ukP)U7Ll_J@bA%WHQC72y zma!f!<(;5Ln{gQVsrLN@SU^?*SuFA!g;+q{mxw%}u5o7+Q2V`@DwX?x+=Vye7|I?gl2mR2maZc9>9W+sA-G{|eeO1~L4Xdbgkq>fr>DnofWt#$Y*B>q!B!ib0Dn z4Jw0dE09i6*2ziZE5&Jnv0lXO&>W`})It*k1uPuKNZEvJS#3M>Ui=t&nO8D%A*!Fo zSUA`qecN02rQ*T_3@y~3dOrwFPithXFQ;XvJTREBBJ5wwPa$-HFtzVE7R2%xtS+Nn z{KYsRD`2TCL4{TqK+CvnADtxP-s{-r(}x%wqEg|!yQ4K z^p!#^=HlvH77>;RFy_yf9V5oL3f=d%Eea~;w|fct72?yVquolfO?15=OK$39tU!(D z)riHttGWaZgKM290$SG5aj*j>n_Y^AnpfBm8B~+D;5DLa^*CnLt-WDJw4AkJ+DNMy zbw&mOUtMo2lyL}yK8TeS{xqd-nWozySNZ+|K~^(nMu@@Vq1aoFELtIvJ|QToQY6wU zmK9@9=ubGR$T{&iE(!1&Vz@6i6+cW>BBLJul5#L3g4ReY%*J1?P;#QpqG2uz4PK2f z=&r-LC{2oe_X_+`X;zK!$07k%Dwvu1=_LPH6%bWXiGgug<9m~MrH3oWVqcEmHkmhM zt-?IYzrLE6mhz)9m(;mpY6{CM(^m1a9K$Tn?9yS`JnXD*N!^K%f&pNIAht^c&tcW!r;hUe1p&-4yOIiZW-d!R0r7CA@OE%Jv1 zSz3shl$^;$hnXpFGFiub!k5efB3_LwGzGt^-rm*Gun*T?(ZGAu_vzLIzj_XU5dewO zZf5YCVh}+SaYn-85rLKi77%m;OQm5Zc{AiiH#AFC6Ff&8FoC@z^x3VTaj5*YBP&2r z;<6H2sC_4`(y5|4wJX6e<6K?ZEJ=WN|2O5xQr93Hl$Aj*ddkCmh=y1en6zn0Gp+B& zu(T)AT59Lh6@o3VVK_$4e_D{0wW>6_dhdcqVs8l08eud4E~?3I1&3G+lKO+B=uc%C4m0Ed)yIg<=n z1l005sgP~9SY`50$fsen^1?hBFj^?g#vv9SGy#{9dB&&MFEb&@I(h!|CsT=6aK=f5 z9?}YuNlm1fje;x{1SeSl9p`t&-k8M(+NLc`MTYrAscZbRF~m8PFtOV$$oPP+pSCLA z`bwaYB`n->H}X>;MuRYgvE!uYJ`GCW%&}ya9{jn_@M=uK08!4>9@_CqHUCK;@E2CA zu*v-^UabHKC~!GLLH)B}#tGTl*^O@X#Rw|NMu?G#2v&y|p%N3>)O8xcnW*EbOZI8KMm_=4D@vYcTj3c8{ z;p(DN|MM8)6bPqeCi`iEj7!mrRB;Kg06Lo@cx$YGMsE?F^D*y|>(|C{$O|eYze$jl zPU62QScX3b2t{=>488t(y;{*Y(X7k4{7ymf#H!`U<}*uxEk~rsHsZIQ2Ve+g<+{?} zxs(^n5QjvN?fiBj=D!FDr^uhS41f|$Wppm^mn=_x9*@>F-*1ZmA~&8ZUzv&$73E}1 zPt*vp7C{lQI6zMWyZGa)z^xEeE*dZFtHux}@wL=okgXq&&-{%Oiyi`mUi{|+%K~@h`LUe}zF@Z*& z3UrX;{fN~77QMC~ha_)b17^`{3nX-Yd}GQN`UrXj`l5Bh@I`~+bcjZ(9YQQ}b{*rz zZvwEgvuy#HvzW=bzaU0Y9Ld_57Fn}5rvh`Rj4}T)0hSAH4MuI?-;0rOS|{88XFGl@ z_hX?et*gS9?@VD@n$UH1zB0z>w^6-v7G;L8mYzhKFfBsY3A98RIPpCy6-N6O46zd8(RI_dN*q1R7tJ#Cjl0>cCymuuY3i>vagG~;m|xkURcBi zk@7`a;j?}XY)lw7;nQJ3_Huvq>)=)jtc@~GNq-+eqkGuqY&)p>$tyg9SY6GGK*L;r zL1Z8--ZH|7H3MeZ%;9ht-C;mlqSKLvz$F6}*2*`Y@>TmmQDh87>?UKc8mGQ*jIsP& zgfVG0$HxJ=@P<}XEx=M_8Cw@!d=-0{fEbMdu{SL5wj(S^bDiHbh~#whRwd`T8=F%Q zJ3I0?kKjkWm5Mv~l0702BYJiY#^t6m(6spjn3a%42CZ?ix@f3X1$tek->jQ6+62#3 z^mwcjYiQ3<;Hd5Cgf>!dfUHDZT`RRfJ9)CWf>Jm-D{*NI(>rjs5-x@b$^{#3%bCE@ ziwLj?nPkYO##MhzA?jn}}`O8fEFeR33&>c_c~QA1*zg-zOmQ zq~i{fsq(l&=zQv0~LsJAaD(E&m5b z&)+XBZGNRUO9SDReJO>0?x7SUO*EEg{m8>Wk(JNXjY%QpwfbUYz>Ec3z0_6$XcB}h zaUg0$y>|Fu31KZY@R~C`#2Sd#+C^kLyAon?=cl$i$kL`RD z#^?Kog&2uV=0z~_j|jBfu?sMqfL++yA1cqVNb#aT;|v0`O-Y>%J^EY1jRwqG!ge8_ z{Y5Gvbcc~ig%<=yO-%D*RMbL@2N7JeTkiK8^Chqq8fR~>Dw}jDap`fC(W)usskz$;> z1v<)#*e4sIf|6F-xB}e5e()IjTWThtMqjF4Yf5M#suY8Hovih~JSO|&`~yNGuVK?M zarixPVoZK9*Fo=%klX(o{2JldeatxOe=Ed7kJA)95a@RTE%#bxW3QXir-hHvb!6)@!l73OdxC1c21s`J+3AD*nr>rC(A-=TYgGxI0^>%(=3d@`2v* z!J%bFP2H$9Az=uVFJVg@3dZ+^kiBrqB2<$9M39k2Q%R1gXiLYKbWmd`Z4wA}{#s!< zvWvwHnLa!Amm{+k>eusy84VE2*-cjN%LEy%H141?WwJI=U<*l^AF$^?jt%v*36YzG z8JnPz)wSs;sh5NPe488mW5Fr=7oxK)o7n)LL! z{>O0yvIQ-b0A~*c8uiA7ZCIuYygl|?iG>pxzRB~W1lrJ=Q#^^yrh2zTSQbYngr)Lg zOqOxz^Nan@VvNazW?iNJzfoNHVKXDU?m7~e5oJ7^|LAL95)yG3b;qHneE*%(EeD}B z(U#pAywkswbIFYFsTYev^Ivoa?+7uvhPjD0ru%Eo0<=n@Wy#{U^8Kbz#xVBB!5I=R zHR~tyH=dn}LzA0Q&R$fs28olaX=UIl__XvnlYspQf2+b;5$LkjBDi!wnAKRc8eDUMODE`X!eCsyjCD=ke45{jsY|RA!Q>$)0#D-X%(rAjLA6bP^Dv$(zwRS zPG!EoQJ|HbD|E{cEpCakqn$LYvbKh_4F#*G{#N<2bW_Xp>V<9>ZdBp|W)?4h6#*6b z=oF-nTJ`r-Fe5i;CrKFY3WVHP1wOTQ8JN)HoiMA77yJ#tjs%>pR|M?eshA>EH+Upjbb zgVQYntsoPMtF@ZpM+!1(qF2;Z@a{OIv^mwpm+S9~L(*4p?@6AYBFK6R6t)xm8+vEF z8qb(Vjy~3F)HibFlBQD>GxK6l%K>w{a9-s)LCU%jgPUG20f-ngzqC9k*o!W+aL_?E z!F6EqhRdUHMf4mcC^@nPP!w#jF!fbxIah#?B~4b@FH(D@c^C&rU=9i`^N7A6mPw8z zqwZi`1eLW=%Rp`w8%UVa8`dHNOY{mt4rBI z7|i2rU4I}boX4)2u#B)H0;Hb~=My_$YY`)3ilGWhe72CN>9GpH5*J&aas(P-5N=p- zM++V>%cj!Pq%A)VtCM21aKZ-A{7tTAc*CGif&5WDoTik@|H(LZgej>iSp*XnHGwMG zb`0zO>*543a+6QbMJsJ{9bSz%=q{N(`rbn9h%__WaB^K2`D)g5l|EUc#~3yO;)cPg z(6EoTE0j{q?;Y7whKzV-{Fcv45O29xJ6 zzui3Lmj-D(^sM`YMS>yz7}mqqgatoI!8CLUvkW`cz7hT92gA^|kw%R(NTYPrZcPM`d+ zKOSq#F%&j@BE2@=o605K1INWm{I>g2r6F8&;TsW;yyABd{eLMEme{IAAZAoZSXhe0>_!8i~r4%f*saKjJ*vvSKq z3Y(-TPymfY>`!EOSD3rD@@l5}MUIaYr!j!0NitR5r2r_B;Ht{iKxwl~(|xc4){w-sN(P-C@?`|k$;=>CT^eH?;-_ze+IPmWA@ef7FYzx4vNp)lEHwM3 zRXE0f-N)xouSn%JoLQt_FB23sIKpX5QicDyKP1J*nXr4qkvqq%C4m1)|fVv&`0L^TjejRY*}oR`8hvYpv8x> zn#}`wzCsZ7-{-PES6%8W^=4VX?2@)+O^Xq9|H{+@LM#VNFwv4ij1|HxHH<=OT!aJv z8H3sjZ*2XqI5gWeY(wgIrV-jm%^Dn1?-v51`p|&`?eY9dSVRjg-y*~PLl8Y@tDVu( z9aQ?tWXdlpTjZ)-Kc^-I$%HIxXTJAzAnUQ%GJqDuOrFh}32Zb#fnAJ(Iv`fMl3W&i zuu)WLpyg3zWAV7mKdhjJq1{DukNow7Fv~aNFelh@{X#)-5_!{k;f;E=_OGKp43Wqs z{-s&wNfQtT<~8-a(%+bstZF~|LEfYK#fK>b*9f%qkqn>&iR}_bVkl*>tL^{x5U*A` z`gjc0`5(?n1=2>)Ad~*?Q83Fi3|n4Cs|=9kQunAASB33x0ksq{^~{N`oP6^c1*XJ0 z=-j0E%$o`aAvog_W8v-|HJA6IbeSU2&xU^R{n%RtAZsNYs}yF1=fD~Ls4PBZO-05jDP87dkmVN5gf6(jAdnWhY1)GD-zs=G#Kcjd zR#0W(c&w~PiL^8s4%#x7+P2NE{5GEEEW`>@Wa~(*_E^EXKmY zJ-R~wLTe1cNZrD3g;_1-6tXg-4hh=-S#ZQUFcke>+Okw;U=}!Amy>@k(7b8a4gzh- zau6c}Cknz;N(UtUrd8ndqrqTaNSgXOsM(t-Px(vyYixc$Z$S23dRL3N@8Vg2*;-qc{tD=#v;qOAR&O+D3%h zom44aE8p53hhKI00X3?YvvsIKH>tG3(Ans2@-oAWD>=xovzwt2xAH1sk{w!D5>U2m ziPC1iG^b*$lveml^u-cGI$`rHJ=tY(A;!w=x+Z=V{(7OtBsrRLXzQ3C8iOiqb-6_M z`vn>?8G1@8eA^uWRs~YMF?Q4mF)wHYiYxt|T>w_2x#64<@?}sL&UlRvmTUoM45U*o z_5rk9m*g@N5c{<9NNLa+J6yxlQ)`D{fpU%?#e@V&Odr;YGni~8!kSK$+IBtic zMyACfC$qf(MXqlaWTYoW&Xvd(&;)-%uw|OHYi-Lc_E*0L2ES=Kn?Jt9H|X86#;S+R zR{69)0~zg@?qKxtQUUl8Ltrsao#WGjEU#3bt`ivee;`C0beD|%#nHR$aO>d4&Vzin z%uFhaw$PM+02Y;xqg?FcgADHrjJhBly5TreB*DhMbvLPwCl#4*%RhIFaPBX=bp$NI ztFtMu3M)4uW~DH4F}(_?WejUsg>1yi_Xm`eVeH!HU~ztq|GzkSZ5)tJY@9zB zd(6=#&Qj}rAk1hpwz`ZNoBuFII5Jz#g!;MQe7@}{HC49~vT5sO(f2Zm>A=ZBwySib01!@Z{v zXw2W@pwNdKtPHY|5r5j|O)Fy@#rlOj!;RQT1%UPLjYFd?WH<0+gCdHT73m~PD^DCT z=6YTZH!7TYD#_O^lYVv_Qw7s))#uU};@F5Lz5WA1Rxw<*P=IO+6X&0U!R!b!sN72Y zYQ0i^pM8INJ~S%HD}% zDnhlt@;L9-%ta#d%g|iyKl~clh|HVoipw^6Zxf3^Xpd2oQpy7itl{RUx9fck= zfY#?f)*n(eT9edM%gFrk?<|Oww^hjV(i<6*TZiesvC0stbi$tf<2zIdMQjm}QC~ zC!B5iP>@v->K17p{v*W7JdU;90^j$WR4ArOTE+F(3bK6Y_}OH}pc0>#epgFAQ1^vP zD=1xsP$M|H2BvK-yJy0Os1axz6FWW0Vk;CaA{z}kGRw|iMPh}4O;|8uNcn{@%Y)|p zToa65>H&d9x3DFLnUpLw-`57%3XbJl858j11X=Pj(H->-g+i^4V%;@$m&gD&GD(Tc z$)tV}0+UiGb^cFe;>R*1?PnN-{E&8lE&V?O&52Warz90{1yhnRk7(P`FbzHk4xE$2 z(r1iSB8Y^A*?>j0Pk$zLW3_p-qDd#Hg@Sf**akREy2_pPmk4BQtGsod4!*-lG~SM`+fTcCOjD+rCgblgOB@&Rn&Bo3Ow z@H2bjBPu1LC?evX?-)W~!a%G(G&zot#ZBX(HL*7gcE97(8Ot8a zDqNZG6UPR3Ndr>qhYPdV^4O%Jj%>6b8!{%9jl&8<9882Rl;W}#sO>%Y z#ZpE9t!B7w8a_qM7_g|JqXyF+UrO8xtzydp#|71owF=MBRYm<5`;U~mQH%5R?6UG7 z$NAAHFfPn*|0>Xu)>#7VUu65wVhjmfc3UuKGCCC|l@zgkOo?iMJ6LX zm;^5OZSPJ6ga+tb{2(C~P?KC$!eK$sGZH@hE%(`b;aKzz%sXe@&pY&`Mwlhoj&}cX z1R|E9V$#Msvali7d;&7-Vf^Q`jo&EHXesHCH6*cNKrOfQ7@T`&&~Jq?^ONp2!Ynkh zEd#lBk(p1ca-<4o8Vq%QlM=F`m6fv%ni_jaP{g%WS~xSrJh6QM2`=C7jy*AB)&=r! z3bC0bi6TY86k#gt&&lJ9WgDy06IjXTq8nkx_~X^u{V)NRDyRCTrN3K9lq&OM?P|{z zWCeq;lDgGRY!qtole=KW$oI+zW2!x&YHC5YpQbnSp{=CM?oi% z3N!qo-azwEfVC@S!(eWSo`O_^#D`%l!NwC~HDa*crr6IB7}XbgkDv(+n0d7I8_Bqj z%418Nm2)`AuMlF@%0^1;YyUcq>%zD7YB=rFbb{9A&oL^LM5qLR#?KM|OUpUgg&JpV$uyofj)lkM$_i?87A zylUQ%IgpK5YvxQ~r2ZNwoZ_d<8#46|m=zy$iWLm7uxyJ*@)!($s8Rw30kW$Q=dq0I z3!>Q6yo-yp2PWSa#uz5jDyukF=BEj>6p;2<*;n8Dpg=2jh0K~`W#pyUpY+`lCn|mB z)Re!n91><^$r1z|=?<$8pJPD#|ZKL7ZKD zDfj8IN9IzDf8<{Z!_z#GQc5o!g0fTRJu5Dq%;gy059WhV7a?+32EE7|y$HWOZ{9XQRo>#j{EI@c%C9!(s);K@Yd z*+l0RiNv}@VoM_NO2U1a$oT0ExBO&`oKtZ`>u{ zB+~zqocB#4?bRjvIem3~TH?_(_exsg^|b%C{3YIQ^?P5M`%7BJ1%IiTb-|};83UNR z4Ei!HeL%&aZ_?TfxU^!>^z@7Y$P9zd`{%BXr?f3Csl2arY*Fs%AMEPrHl{gO>-3jQ z;F|uO=C))wH|tyc{*d1To%@-7e`feY`aQt8=Izcs-OeR{iCc`D(UrjGI#+`m*wrQb z;)dgf;*x#9anIwgf#<-x(%pcY(%p5Zxa6RMbeE_j3yFkFpW|FwgL4@RoNF`JxvWLb zWiE8C-Ad=$E_UwJM(0jh>RgAd&b8m+Tqj&dexJ6-xz4+tJEPgT)8BB;?{)6Xcbq%> zfOBWMU2qvjq3Rw1k`4-X#Yo z)7<}_>XJ8{-A%;__oIGU?&dGD-0%rm?v^20 z?#ExWaknINZOLMbI(%sBa>8_?f^^%|F&L)o!Cfc}$ zz3Hz0Xu5l(opUP)+Y5X)zc1z4&?nu^<#*FQ=SC6dG~AB!T=EuZlguZs4#YXoxfxy4 zT~p^Y*G##l_riZ=x?56{>Gt3^xt=`Wn)zMBx24qSUVb-ylJ1)CPjj~u*9z+GE}r+| z@^A;_w~%<=Z{wz)mT-r89}j+%=Sf_~`4pGu258juY#Y~%zoxbcx0&~)lyyv}71N69ns zHGSR2b>+P${${DG~txrE!wvnTij{Pg8n$M@N)>l2yoRG!JbnXYDcW}ruNn+!L7 z1ZAwsf>_{NGu_O-S#DOZELS_&xrd2kIqhyB@zfv5at+N{Za&x}yR+PE(t3*DpTbMa z$B;fe2@5Be;g$@}bVCU<3O9{3hw~hRYn+(r@_Ftek4<007Z<0ySNXk!Hok}F?BUM6 zNciRO$rhgL2WGli#5;2VJlmB#H8{5ueELf2X0LONi7fXf?{neJ=?fF?b>e7lp8=0# zxEioobqQC$BjKt@a}RV_LtbaY8-vNy4{^<4$>Z<`{-)O?+>AvD(nz@1h;!!3gqt$Hh;!5U_62b-=>^~PB)(AzR|mGdFZ}X6{6e_@ z!T%gE@#6dZUehk&KH=H)3EwYDa|`&kgmTwrIJb!3f8kpr&lQ_!OA1H2n|L<;8~+SS zJNUgvcwM?Xjk44nPIrGJ%&a5n?suZ)Qs~R~Z@^aZzE^%1(v}aTyV*PVb|HDY0H^kR z8Eu!obQZjj90hGhJNFggW>CJ_@WEli9>BdtJT=qsGm-Xm0dcIO{ZC7G?|?5kE!`c3 zo;AG_Zv9E;meZ#+4o-K6!Isb`o#c5Oca(UR^PYqbHC>%s&+q!)&OJ#PmM;J!?j-5Y z;JM^D@8qem#<}B!Ti-6-H9&{jfuv76tfjs*W+mIHuXFAsewyCSa!vHJ8C~Fu2K?QU z>6+WX?-O`d^1Lp~J>9#LYg&=znpbDJnbhU1QJvuTPOcqwI6cwc&EP#r-PO={JWswJ zp^j%Cb?z~KAD|2mk+1raPOgElod~m0;d(kZT=f9Ge!|mtqo2X`#C65>#%17=y(kmz z--PKv9&4)^^LW+_hj$hy+&SP&PLjWV)C=QmSAByX%|B0rmpZr$`97NXSG>{5&Fb&m zpP+ri;!Z9deC}z^{gQ9*fi2|yK7J>+cA{N(a(l`5j5(d$Kfrq`UHGgw&u+Z;<@r6H zHH(O|2lUt^`jGx7q`9#@{cd}=teHC9LmBb=NKGdl1PiFXw5 zLvgdHr!hPyl8>wSz9fUZoS;rtq`Q3Zo4{rdqn?k^zIdOEzw5!o2h(`3!Zodi&t7-# z0A-q0=-fe`H9Kg7(Cq_ZTNCb9!pvQ2{IrNLBYCeUA4~cD^jyYG-ao>x+WsH-{TtjX zxF+0;p~OWTHMr!`gd0d5&7A1m{d}t>Zx4Cr9;6%%(7GPKkC6YdeA|YbO&ZJTvzHA^ zxP0=yjyx?H&DgLKp2XG2QI;gGabm(1;paKUI|n+F&QRPE=skw#C|o|SrUd`E^%?LU zY4!!Hqa4MATS0mY_&o>L*bX|bBj2>;ByL7C?G1brZU?SuJN*RXbOZfCQy1|v^1&YJ zly=iJ09U~8A!%;LaQa^S&)QGj^(Jk~SOXtTC#`bQ+{?FS+yTW2Pu4?^`a_mS$rw$w z#B*E1Eg72bYWp*W+@t2Kjpo~oyC0sSXC*lr~e}(@u z&*T{V=EFA$@{rAMcrQ83xjy(i4_du|AIT;c^ZRn#XmI+?4I{}A8O{495f za7|m%-HK6J?g+n2_;!qEg1)*P?u34WEoo%lz;kxbOqZqSaOYN%ujwVw1UX=O0{H|w zEe4y#bE}@GAs;j{-tfEWHu8gvS3|ue-+-1p)&D0?Qg?e3ZXteVGhQ_Gb*_T=1CK%b znZ2PGSnV|CBIIX!9bx&M)Yw9QaV}Ws{|`A=O}Gber{L;4Ku_vqy>QE^n`8&lg9m2yfu6&tvvX~1?$7s* zgqw^1#u3o{UHJIxgj+G#xxsvEY0ouB)7=yJS%jNS-j2dIOZZ*0hxSifU&-(3{2oS_ z+BebzkIW>k(Y%YsNzxd>?^)T*sdyIRJo#2%-Mrf51C7%ih^I5|4BTw!wu$G-?Wu_#x1X7zKH+n z`Ml##Y5ZMreb&V_|C{k2Y)zwc`w2Uj-%YPDuD^}^xdVDVn&tivyoUO3I;X2^z97wM z-tLUeK7|D{)JX;{Q7QBMWWieK)SA9sJKT;BTk774!jTk}sv9 z{Ggk5)zq8-z<}TuW)~4=^O?+R7gHAJc z@LtF7hVJed@k!2lnld-={RHnCQ!}XxvXV@8pngs=zvuUo{oUP?>TYfcb=RKnjeWbj z<=Nfba?)E&_*(of=eY#@`=r}Q{IwIodUSW|8F1H=<^sMSqCBS){w3A-M(TTA7k3-@ zddhPL&u75TrTnMidf|HDl3OW1Zh8;sh|E?CUHX8}Y@jW{6ZN#=h9e2LpST}73Ed9U z&y$~a_<7_2VW9b3`j8s@Cgm3M8+U@V{)-b0PNR$s3li?>8oo6X5GP$r^B1&>!{{jZ zUUQUoyHx!m?eiYy;a$;L5Pro8=X&t{yZpX~GUVWT@hmEn;v}YuQ)~!{wu=2 zop8OuBtMM+zk=U`d0s1waHIJ>3^(f-^89|B%RV6Rs%_ zJpt{0Hhv2g_8oKqbNPNC)6G4Of7iw}e@4HMPkEr#j|ejne^ZHXsKTEEE&EgFVAZ&$ zdx)#2bC*z`8h<78FX^A|`tW`wZZUqAk0xGp1ogC~B_q<^O0Y4+HKU}1TfWG-WsTrG z*LT1lmX5^s}#M?j{sL9B5!}tyRCP(86 zaWklgWDmYk{-kKY@BZKoq67J=VSFtk&6<8^^fvzS}&2Y(0G75X>ViMYUuV46F`_j+hjGn~BOMiBN0 z*quD*_Rn%xj7EMa$#N%%yD3ROb77iWhrBb8IBN*g zgZx+XtR`smxX)sV)VAu$1>(;&9mYC6`8KC z5Zx_U9~?iEO<$zDES`OF?~vB|1;|4M;`;KtAMQHbHMmP~Rrp;-etYtKlKgk= zfzD`Dy1RjI=knX*vM+!Z#m>#o@8BMDY3>>7dC}5@d#Ztc8u#Qv+9A)T3mA9Xkhkl2 z-oq23HKU7d+Mea=lu&{>X$SPg#J7zwbs5f0*y2fJYxc7?2l{uL&9^IE7f##GwV9K1+(FahK}xuA&e=|VdWM_<&{f0ND7Ddzx{?k&lFfTFV{akPiMVQ9Ju?1M;t2Dz?a05(3w}&| zGm!%p@jXc!t~m*v6)*X$8Ae`d^Ud!=1D;FqvlxfymTYGHoln0qhxnSwXH$D zF@yeN-co)y631TRSj2NBV+4M0Cq30w@(tW3@ET|WKBKS=dN}HNBY9Aq1JIq|cV40m z{Rg_Huc$xDs(j67Onrwu?8PBoGG0TcE~F2?Nr$$Q@S=4R`A+k+<^p(VK5=wzhd%Wz z*K}7$=K1i&nH}ioPk}$sV=*ymI(RmY_kO3k=C9fY{u)Idk|P*XafdD-Uys5+)YTDw zH|-;y&ydN5J8;r$W?iWnokH_S;x8cWV&qG{XVcc^FQjjI3dh_hJh{B zZ|FNh-$(Ji#L}NPlzPXn+E#zQ&Bb46f5f{m3O^KlaTI>I!XJTt$KYM!TPa-m$Bp9K zKz%RaJ2Vxq%vhRm9h85{tMpIfcLuHp{+eEg9`C2Q)sz|WJ{jb^{;Xl~n<-ZE9o&B5 zltJY=#P6Ppe-H0_q5mt4^~ga<3c~y zgS7Tu>lRS9#3B__c1vN$QQY)0!h` zZUk|U#w9Zby1Hr13&?M8=%+QmUJPM1jOj_n2+?nPGf!wVU1JP^tPk~193lVE zZ#w0YPB47$#qWvGm$t0_K;h@$SADv8tAXEhaT7^5tkHZpkaE!9d_tJ^#LpNd{l;{7 zWyaxaT|4}qhC3J6hcMqJk5xPe>YaQubxvLkmZY3DgRgbdcrWC84bOa@$)4c7u66U~ zLWgP2eFwkuY5z_8s7v%dCHR>)ioA(W$=@dGbQ+laG!jSX_ZM5<#mBYO-zIR?;|`wn zlzsX;gx!yyW*mH20}a$(_wao^ZYyp#Zr*zMpE%y&cW7rm^4}Z(1JS=Jzv2_()A%=| zhTm@wbbT+N{i5s97*>Z|KNo+OQjUc@8_EWR|8p z^3s&%hD7l-wI@GqX{)CZ*E4)yN4SmjKdj1fz4{H@ z6kHq1HQG6L(vTnM_FBT-%`9!c3mahh(@Mo(~4Ll#JCjDuwtyi{jGvT$J&okes0_*DBdxY6`3-!)i zVavbakrl)R9k+vPZEBWez6SF;LHrT@9#cF~emCg1aN^s{?^(y$m+&rnEyh!(A<6xe`43DN_u~GH zo8K3D_CZ%&kmlHql-y6;^C|cL@|*VT=F=W->x|CsEavfNyW}5uDvqo9y=h;%+sNEm zaqPb<&D{;Au=__62kH0W{a__}jn^r=hc-M9Ceb^B*E~yIf0pK+e;vK`J$%RiLiw*v zlm5dk;`j4kr@4b)p|@#DcMINT4H9g_!F0EbFw4oqWBZ)T1KYXVxgwrV@GR$9jZ5Ox zewwyl>vkfrKg9d?Dec`3esBA#y?c3Nd$)z>W8j-{FKub>w(;D=MwAzyMMv>$d$*YP z_1C4lHSqmf+W3|X)-P{jod|p_Zp9Gj3Ju=l_i~=kL&K%@@Jiow_tL<0_Za!y#B<>o z=K8~#^V7zjDntjs^BLT)>Jsjk%{a=aRZT8i!X-hxGw|?$d z3)oKtUoFD#Gsn@ntWUT#Q_!=K?oQmof%M-zpN4myYDai}{}FfarZmU?Ik)>NJG@aNBUZaj)Z+;*uMU zZw|GEzU1o|7<;zd9P0WwzYjTh9Uk~#erKoo#}f_65NFblOrei;ZQPANVEqbrEADpO z9k@l%>hswJ?wQ32cQ^0%;+|VbIe3o6jmMSZmN(F5cuvBt9F}lXd9GgR+%I@e$IZlT z=t&#q`7rJ=+)K>&Hupz=)FO}p)ixUr>^NYlT`_4$rIk+@2`-{61kNo?uiHGlcB(ZMq*=~KJ zz`Z!92pw#Z+c>PqZCX&~IuiH62?=*P&$Dpn;kx0Hv?J-0Li^gaya~5?OqP3T6i?{6VK;kSiR(MK&0RCy z%{*_#Z35p`nCV_7U$^sq2X5y|;>B&p?Z7P|?z_S7#pU3};+Bnu2H@lQ{X&AW_0Dvq z{4U3>J(}g#;2LqOaVtn;W$%PrMLT+7ApHQ(M&8%(TuZy2M3||#U*M+WX6m_vI>04y zTNa^DZ-l>=Cfv*Pt=p^MuW7Uup1XKoM?bOtBy;86j19Prn`oOmXrnw|BHV+-_b~1; z+y?rC$9Znzxf%Bqzh}Z*vydydk++xoqR*!d@4)SxmT?9xIfeVf#?6=-ot%9 zGSmGp&%fYyZamL@#Pd_!-*7Vvy0}^N>0k2uAGmLD|G~BBLBy|V186h2j=0ltXW>?K zWt<+I?pEPmz^%qL4x_zKq`l*6FX-yd!(TVtw{YLVeGhjD?sD9gBlK;@o!fTOxtEVY z|GCf~x0Ake7jE4l^t8AaaT{BD^x0sdH0_{};IFxS6;Iacl9nhBnklxYf89YG@A&;rY3lZUxUw{7k|O3Dp55FAem{l3U*ev_EyHa_IkgG5 zir+8q@8-6l{942B7je4^ySts)-Q5nJ+jnb}3Y3Kc z6*C3}s-XfEsrU1U;C65K)cmo3eCzi+=XvgV&Uv2m_B(I4YTGN}s&KC125usjc_mci zHofksx>Hz{-_xf5g*rjHlM}=-dKbGG(F@fDC|@am9W4%b{3f+myq69M_vjDs2%XcE z-$}l?a-1Qv$c;9iw^;hcEiR6AWIcD{xH76A(XdNCIMHq#4d8uQ= z(>&ItPUz;&a7TG8Cx16qtA7&8KiQH||IkyyPal@|R;xcO>>MMUaVW+_#K!CM`2LgW zb#LW`_#1g)Dt$T{R_ZJ8OQ3#JUYN;mHs+!f^RNJmumqnh|I4^nVGZ`^yH|cAKUDlh zept)B4i)(Yq4zUG!g_j7jWR>_{dip1Om4+?l%X7z*oT&rL&E{`5RTv&(mV7&aDv{o zQvYM=&~S<#Lry)BMKmYct*)tAYHgD=#ib`oYkf-EGSWykB-r9L&xN?(CDDv%?Rc8p zNk;P(?aFfpI*0qd__%(i=g@e?9PjnK&`h?x<^8LVQ%KX>-mw3m9lw4h^nMX9{F!=Q zUdI`n$3u4u z5tGqZF+NNsd-MyZlf8e;H`wUCjepvInaQu_zPbqULUmJtItq#5>MZqPgL-hbd*))W z{C#R@D5cNC0xUxNNOj>@aa4D9q5GkJfia9Mnfp-xW3jq)xjJ>Vx|M9@PR&r~q77Bn zGAt3!GOWNV#P(YMu>ZNRhF)j=Vw`&|eH|Kvo8(UH(tjY=^K0qRzaTdweQQ|QN^VE$ zzIbKC%eyS&F2`WIt!#E5_72tmHRGj!($hXl9a%5k3F%0RZ-cm2`prHZz#$w#d51oE zlW}*l{jU7CTmIWG|B>Ctm47nJog*8C*HkF|-fPP(`OkMxk!iGDQU0q7!ZBf!%6(p!OdPxac>RaTV84um667 zyouXrKRQ(ZOBt{BoDIU=B3xxZOXj#6?+*=4kF9MuZQKB@P3ne>aex&2vvz2>BaC}^ zfJb3?#+o33IrEf>W?xK)9TI3rQg?e(QK5jHG6|EyKbG`4=!$Ub) ziGApOwJ;nYd%kWwC-OH|w5Qv61pOFlgi|Y=_o&0N7D}QguqxSmM$*1)j#-QU7MN{l8bXgwy*{QP@sC zvn$K!<@)u}7}B1P?6c#p#6BEA-#3fGA@T^0VfQz7gcIZ`^nS~_2(qV@?ML?gRDLF7 z>a~hLsI%#5bfQI@vyRN-49+3VFQx50PwzUZ4b#3}q+dpk&5%uKyU?Aou3rA@v%c+F z{>y%2Xt?UW%0I|&-!cA5zk!>$jXUW5a(=i+_I#iHUOg;4pg+PB3?Rm@TG)B7`)=C8 z$X*#nAAtsONlw#7qFx+}__Y{k7)6djdf{_n99fK%^%$M#!bI-L$j;I)m}C54zJ5Xp z`(GHf+Q>Lrr$1AFT-f_Pqe=UVseUsZGcg;3^?#H44s*HNy`K*6Vz8YT=WhDP^{e`} z@sjoz%~!O)uC=-+)uI1UtN$ULRheO-R5eJjBcTpR9#=*Vvtj*aBQ|3zwxbN?s6@MaI^5fNQaMHUCgmSl2hx?*yNTT9P zZ(FdUJ^EAsvYB7@qqjpRS^1S6#^l6{eg@}o9yQ0MU%cX{g^zlA;)3w+3m+HVa~W50 z4etg05Ar5%qy4nAT`iWJglL(dn7g*(FOwa(xk z`2dgb1QkEt5wi7fhXH!tJasU#?eByUWZ$QTgd%bj#$X(Z(euM0VItZ4W#jQ=GUZyWnX0OVQu$_My%2A2f;vJzH z`{;GM%R(Fn=!eiiPa<)+EYu^ae_CdgeO&*f@4pi&%u5_`Z)$>PUw9{UU3%wfJ^Jft zT<#t9@;mwo@8~DI6KZa~6KWs66XN~vggUbR?mHnd%QH&KLIZhBI45ulF~pHXGty{( zzbth0cn-SIeM{Otl*XfFA&17rW%gy4g=Vy%^KUx|JJscO@X!8wSi{g3U=3nPSI zgzAIJft$t4x7)z!{W1^kLJde>&AgA5R zUNZh6%x?a}_-(~0#Bg#*e)#D~?_z8Sr_Sbw8tqlBHY)z1_OV}^b64AQMH_To`>6eF z)Q&bSf6n-tvCs|1_qQ0|A8lNJyzzaqeVX}!S=!+l+TlX&?pWIgkmYW;&;CBh{yxh7=70G5mhj`#e;NMv!gs@wY5#XP zxcb}Shs(bme)RrdhC_?r3de8#Rd{#&{}cYBxSi-SBrO-wwwfza7fo z_)hq1i_Xso`y47qZV7u(VN!GV{%_m6`93H+c9NSb7{w`AxuDX5=H_&4}#Jk3TdY#L0 za<}rzugYS{Q`Uc14|(3cUHe;Vhlf~Zc&M!&9cs+g)FIwKIwU$qhWe(Fp~1XP^5myN zlZ~y7x5kH-566b)A)gMZA%7lPcZ>}cFZs?G+&^(x{!vff@tn%9+N1rg5bn_*p!16F za@}`2?K>emBR}NO?LBAEobX*!zBAcW?YkcLT_5_+^weSB=B?+9o9RcP7D+Uq9?|}a zN1pKn1IYWbHVh+Bgi)v&SrBT+8)KML5b97rtso@4zvRM#(C~Ue7~`ICD8@ug##Btl zOti07|GcICVcU11o2{S0pq-G?{}^tMDVlrKKWN>e{@J1aA=@_aYswE5K|N$s!ffHq zMfy^HNLA;DQhL{gd~Ha6m`7iLT!*lmJY$w;VG+NohxuU%xeT$x`C%2g2Gu?J>aTqD zm%6K_FjW5Pg^+w`T%+GShrUq)k^SHEh37)=m&`?c{WPiTp+Vl-M?2GKxS8}|L+!t z+Wx{2e^?mmHn0n~unTvv7sv+g6@GgJ$8Z9t5JMbEv_EA3_OnsRuGiV$$Sh=kZ(@I= zaWwmPJo}eyxxoIt!nVE6{w3SEt9BHIX5pmKi7aBmtHv36-TQ3VOU40e^1?aphOy$t z?oE*Or`f%Wy|3lo7h0CS7|y#V<+*8Gq*s2RACu2MU2S~8xeeX3#BYxH(R21?)GTNJ zul9_??Bs+nQ^L-82HC)USr}Jw4L5KTw{Zve@bUPs^4x{)x4dIy^#yZR^#86%f3@@r ztL3Eh(^K^Haq0K|tHu|F2f}%T+fi8|VW*;{$f}`uEH9u#X7707ZK(HMLey5;|j{Ga_$``a@# zl(|-pO6&xVmBP=rxP_iO(#jvm=0ov8RAKNNH4PWtW{c@*8K zDbT);e`;gaO?zrnCrY&Mv$XG1hJ~2tl&kM|-BLGUj4&pm@@3_!(fggm{ynR$zF?f2 z?A9;HkXh~=*|?JZi)LZ9pmh%WcM|&-ZJt|od{~$)ys4OunTTby|Cmj$b3Kl^^inkR zh+Drt2DpUs%Dt3fV3C9Xs?xc4^)Byp zAODv7)*5~L>G(i;wf-Q#Lr5+6?H8(-aD@99vL))J3BLUtd0>{c1N=^*LLHgi`ErQS zdtUZTvNuS>^V36;-i)ZdNt2z(;tbB=JgT&*7s=}PtzjguB6cY+TqAFw?yNG5M2E7G zQbzQrqcO(X4F#cQrMAD&b=M->KC(d?3S8%x{r329)9-KN4hGkCN8`;$&98EI^vHu# z*dAo(hsrh?^#P)~C8}eZ7PCRn!ZvB1rtP1k4=_jDKT;onUZcF$if0^kw>-OF97Zea zU?fTgr!@XzsclllS4DS_#_Wwimzc%4x``@=|ov&;z zX8)6IzD4Cv$6McF4pvxw|JfOJWHBa2?q3WG72kO|^!$S|!rj|wJc^9$gXxj`%ktD% zb_#tq=3@6ppVPK_hWmL&4(H9{1p17SBN3z>}RW`OilD0*zEx|IxMi+-_ zhb^t3*A`gb$pm+jY#9AYIN+W`ID%t1fm4Vfj`l+H|6|SnPcZ+FZgx#% z+hh&Y#unY-cx`O2Dyh6~`&zp-{-2|Uqj`ml=HtF5^m6MqJ2j+{=~*Z7rK#= z_UwJ>N8@q%Z@%#-d9HBFCM5-xz`N)=f_4 zHx<)S^`X2>RzGHc(`VD?q7?Bf%BT9Ee)Q8iVsQMAd*>V8JEHtI&wYzf5%TqI^vUU! zKi8l6Dtny10;{kFYq1VB3FSYd{HLPuz5mhP|9J18jLLsB|FPbE8?hN%u^nY7N4tAF zrYU2Sl>ZA)%T(sN@1}g^l(WVi-v4s%f3^2dwl4PmOT7PC-oM}Ve7ZQAo7vg>#p1A! zJb*(uf@3&=Q^-sX1)UBk6`6SQCb1>qj|13bbL z#8zv+FhH+csx8@OEPtUqgTeM|q4w)_?H5_&dhJU02)_>XNFa#@jPRQxjKUZ^9e=lf zsY(BayL~acb-8wawRWED=FVv2v)cCDCU)ytcIyRpEZK6M-Abmo(`4H=ejh#;iiI-~ z6+idg{;~xH#`3S>ShzumsDn0;{kF?FZ#QZDFT= zT^G7HsoRl74vqKaKQup-|K!Wov+`e)G@=bHx9lkq&N{5eMx=WRq`x3+rgtgFo!Cm> zj@%aIcZl*kT=|`l7s~imSuapdRw5?f>?04LI>D~rq+H<;clNXT?en$mGnC^umE*yw?R*!e=plIaW^ zV3&2t^e%QmC(hB&BS+67gKpHkuKbHfT)d*SGW83^YlpsvbT)KE{g1+M(eEzfDk{G7 zLbyiWz)iF(&mGEjC!)1}-O77rwD&*K`$yw@#y`=lu4_RnJ%zOU+O{bFf5rCy2|KLa zIKVfQFJa!nJv_i8Ji!3+7KNQ3Nq^rr@H=ILJ8FBQI}UTT{FD7#%cf+XCfKJ*vVrYAOgLjO4#k*=$(V}iXy0P~ zeTVsXvdeh_-Dla?7ueTiuAluc%qF&UGg@}Df7!h$c6u6ZXkq`(6wYkSMJdwL+=F@a zu1&@kHWY;g^hL-W7WPW>-%HJZvxArLtKwcpu0V`^yoy|dYI=MMyMC6leYw{m>3I#8 zUJUh4v|B<9O;FI?AdcWU@&B*?xGC_X4{$`i&kM7;R|9;I<6y-t~r;{^Q_8s3n$ZPK_&TFDr{zTdnU z;$#xdNTU;3oWVJq$3%^SqGvKWiH>0!0{wF&Kxa{VgUZVlsL^vKEo-`Jz5B*|L0Sm`TpYT$Ca` zUirs7de?B}f6mabfW8PHum5|juQXEm-=+L3Gx6iff0MFtMH#uSY>*8X)K3@e4;e0f z#>1ER{W7dV_PYgP4cYG7cA)bu-v-^@X$Dz(PJPg*Z_%`0`QNSlZ}9zH`*{7Idn!NU z{XDM@7S=kf$3|>MRL-OH_t?z0mAlvb-cFXG9F?eAZVr_^fI~Qf!S$Qt&7acihD+NV zdyVL)@bUV;$G-na<^L`1527}@4)yc|lE<}Ehlhm~QFw?WiDsnHi7eXLu^o@~1@^Ol zcWV>6*HY;u6K`sN zXG#AY_wAB?)FXi;8iae(Z*Job?%@F*;Ry!NF1>^E|DN?PzQ6DL@%(?3Rt(PnXQW@) zZQ|N8T;J#~t4&ex?AyV%Il9-oTu*)z*#u55fY?D?K{?`uQDT=$n^9u{B` zmS7oHU=@0QZ;XWO`8-=Syc*Wh*I_+2Vl#Gsp(Jc2x1+Dom^E3BO6gGZxv2 zF5`pU{G+k^!F7mJ>`U^z6F7x9dcJAj-9HQsyA98u|VE(DO(2Hg_koID^V> zy%zR-+g|@KOtAi2KZD=1^@Zo@7jYS#`Wap5Mh02raMiVIxPhCvjXSu9zCW-(zWr+G zdDR*j?r8ntBk~Cbkhj>kA3Zb-BS)YJqY#_p43`<^Kk0SqusE{cDh>`$oi7fkRX%X4P?x7${*U}@2r5G7$b~{7+jb1uJ`M{jv1w)6J6-O zRLZt14O!$al!nIDrJ-p;X=t8PYOg?PXq{3TQb><24Q-?ORhNdz!kdcen29vMRH`)0 zrgz;E_R-QXmtKn8U11}GZp`CX)l(W4kc$wbFCmwq`o45smo8^Vtl(aSWUX|e9tr%Q zU{Yw2&NZ&BMS4nUSVyi$6wl5b(lx&{Y~8x<{{^!F_3TK4BeRD=Qa{BY(@XBf7$BUo7F?ZcK1~2DD!`qNFX zhM%3aR`>P%5HoN1pL&Le*}m7whx*PR8r!*UKFyx5#MzM{`Mx>BIiCtOZ$2Mt%`?Uw zhE_McFzlN#J^ande;3MxU5*1C|1SJsWFXVhsu-x zL)bHVdZ-vXHS9h(H5_}RH2j_KQ29&k*l&i0eZuJZMrn9=uQC)$!=blkhaYYG_u&Bl zzE78iBa8n-_}h-x%-Q~jaCr6HP#u4XUkzvI=g`pM{-#%rW4@{`dNrKqcM&Z+ zhK9>zwD0jMc@5D%$Q$HMbe&ZPA=9A_N*SN@O{00l+pgWgJ@kD;T^zNA#*(7_mVMtZ zecJy2u{1nzz4ztP@Pv%U{{|xWXY~6Y4h?xr!p=S3Q{{K^!!UYmrSiXYXc$4SW8=h8 zL?4BQNo+9nLqh#fU!uMj&LfO*Z5)a*5tA_$HQJims2$m_9oenj(B37;XZr&i`?c?{ zv#-&DR-_iPui4&hXrI77N9QE=`5gB76!!UO_W5}BdB5LG(l&X{Ow7hyJX?QQO7CJ% zN9zyg(H9`crq1%qq=bb<{Hort4vt)g7<~n~3f1BsUoGA}*79?&MUvjYwy(c$ei}7R z(wA}H1@~Q-J~EnHNIum5v*UK^C+*S)EI+GH_bv9+5A`#GJu1T5i1thRfA8!6_2~bh z`=I`xb}M^W|8KAMU!SjOi~j!({eQBRJEd=)MjKkR*Xx9{8C$U(=^fgC{$=#8CSlil zKD`pr+Py4y=A>tl`}kE285RzZhY-`3IYJ&oHUGhJ#fSR;+@~-&uDD6x@9xua#-7iz z0r>ZRRXY(~M>EpsL>6ao4(AcAeUJ8__PxvA+!n$`ewR`4OZ{8<`x-q(rt!4@&;I|Ob2vAl3ep138i7~|qL0@mOWQPQM7?pC!S(NK{WzPyj;v=7 zC&(ms1G)2-a|Z9e7|O{7!dr$FScNrczry}!_ji(AAF}@+v;UE$=g@ea{clY`^L)1c z>K8*RnOe;LC)2-sKr&_rVuK8NVvy z0Oe#QV)T9F0aP11h_9Be(=UcY+((c+`y#uJ{V%==vWDGX`@Z|;7=K6o6y*X*WjNaZ zd(3Z6;1pumS@_geDbE;(;%;|OhkO5N{LT2gGLl6OjmlZm!#|C`sY6k+t%FbZQZ_`bAh(fGhP?qc-)?8PvVoQj&Uf7<^$ z-~7J5dA+tOp)E~r(Z4_L{G*?Z4%xT0Q_t&Pylfx3-^|2p%tiad=!_%tA7s~lHvKN| zA6ewkc+39ZyY~N*Eo}Z)dg_urd{^xMz2KY)Va&q8C$U( zW$62@cSTlWA9_Dk6b_I*U$@?$jMnlFj<IM`=rW-YI(5 zpWaQ39!Kt^-^sg~>)sV=v^BLU^=?MJOV%gU!`kPh_PK#P;x|b&BaKdEaR%qmu1)Vi z=X>g1bic12M)rbw`Kgn^sxrocSir8WM@6^}V z=ye%m=&9$z4f;(qXup&C4TVM@d`d_AIw4n8&^53tFlWl(2vf3PiaHeA>W+NJ#Pi?X0fgbHC=|m}g9&&T!y&0ad zTRnj4Y36VEFG5uAmypX)RiGb4Cd98EgX4bcfmQq_dmnWdxTMXcrNVYK7d0wf@3&=Q-~pso}cSmk-eXjC&)B9 zk;NID!+Bi9Wz-nQuSFbns7C@xG@wzv(WK63CR@}Wt@LN>kJ{8N?QHyxS^C~3(O8oH z#}NGwbw#uXIHxXX(LP@FoNKs&n@CSk5A(11>iE#5d`Ej5?(n~doU)zemqB-vbHoGm;XMA|z{zrI%0pu-{esgof$X@Sq1X+Yp7=v*r#zag;pMHMD zN9O;(E01vZe$|*KITN!n7tx$)DLD@dun5uKiE92!=+E|VFQaE`UJn1?{o6G&>_0~w zb*Nu$UV6Sc@x|t;$=l*xe!-dk(cyLd&e0dnDh#gw`Gfa+wm5W{&+a6vK+Bp=2OXis6HrN8R^2?95(+xMfwq6 z?!FEDxA5N~{bYmr`y+mH3@303F~pHXGul6t{)gh&Fa2cqU2*f=EOKZZFa5WqpMMM4 z>b?}2=58ZfuG59ni7d|G+4%_-UstD05Vmw?KWpyRwH$YLj%Up9Eb>|Yt2%5gg8WyV zDh}t|cM&oAR^RV3y-r%<;*;1WjhmjPElCcJ;~1ZB*`*w~FGXG>Zy-(op82Mm^e%a- z^R9e?CO$oK)>zU9aIP-`+a=?ZQ(uc-e3*EA72R%=sm9%hrZufL-5U4!Xxe{ z7(m|t>Rh7&eNg47j*KHws9#Dp%+gOq#diwAF!zl>5k_GQ#-SJ;AIgJ0%JySroXl`{ zlR0%kmTa10{m&$Q?-G6R3CgwpZ>zp<+xu*kk@g)5V=|_qYPbFcSxv?;lRg)vhzqyw zminh({Uh8Y8m_R{7iw#kYqL?MPRRZw-*}9E0p|H#IW2PUkcU9 ziTCWK|=3apCU-}8S2>dBsulo9UUmy5$%GP3{IMefh5zoc#VH*bga+#9hO zY4v&vTj^b?r+w4y^fE*?X?B|OfK1{3YDc93v z+csyUEMy0wbEWZlbkj36QAw$bQcJ+t>8tF#<&xg>fjxV1IL#{9WQaIPS_HvLkI! z@h$#q>Fcl_8?hN%@yY(ncJ8vs4;9+n%D)%?&l{)YK7g!o{zK#u97B)#?gZIeGc24U zD?akyb!6Y)izk^xGt%fpG^dv(&)^)+Bdv@_{qKwPt`g;Mmj8{TUq!Cp{fO2GbmJPo zolKGp}$`7h;86#4rqH~{$TpNYh zQMO~v3!%~&LY;abo>BK4R|lbCi#iB3Z0lOY+1$}QdA)h}#8UP@8c^w(k1$4f<4}x= zn2f1tzia*f!#}P6-)~O-u(|o;)-97c?#3DH|2gLWhnW9I>tnV((#8eau9$z{=J|;4 ze^EQlE7FqYvoRN?n1=;egmz;99Y>83oHYJ`ZfgoM#u>5;jX$7~ zjo-x9Z#G8JLbh_Jnv6f74blGlCBj*T6)p2zo3Rz! zQHC+zc{y3*8MTOy6%Oj@i8p*7MB@`5k1xnmEf3ijzJJPheYU>jUmc&iuiPVtO3&Db z12}{ui01c?k&!(go#_{?4>-Yn3NgfyL^E3M7lt(1i7d__y{j;!dJ4lide>6@hJ|_I zJpCdD`wiUD99TD5<2(M73-;<~(C?r@o1NrN=$k}) z0H2-1Q@+@qCf9quH7fM}g}%n~|6Pdo-#y^h_o}|gr^kdx^vbXL&$q7(4NvF;7@YqX z);}Bn<4*3@2N~}_RaPi(7=a>;!WfK0MSDSL_pUp3DC1<;aAjIuK6pMGJvYSnQ4ch! zBbvz;_qD#|`>ghTmb3pB4++J>nTW}lis_h%zAvyTeqI!!x&NNOc+vTi)(E{@6lS|t z9<`f0lzYtOE=Bei+TZ#1>(WzXda;*b{_fz2~C7%j^du3!e(lk08E_f;Y_}1v~qtw`NXvU|* z!8xA}KP-GX{NRn3!&UFGh5fuFdbYN40l5rm?eYq86}k>i~u=gvBtHv_!sjF zyPnR!SJe2wOxM?=YEnViNLG`vY1XIFw_-cWP>xFM!vP$^5gfw_oI+J?K{$0OKb(9& zKm7Df>((ZW4}U$&d1tQ0QF-{&VbArK%x{eiyRoluboiU5mqOCDW*jh=_5=Tc^8E{+ z^54uO!=X0{!;iKUI)_pl?D~~k+?z@Kvc!VbyK;HkR{@qq!|3`uS9|fU{>^A3^ zA+y}MyUr#*njf0bywF$`TCMd@Ax&>X)%g67{na1CFky{A5n?OzL-nluFp6HcTHG*( zJ`N4X?Xf`u^(f}oVmy5!ITdO3&2(}mQXNn0p04H5_;-KU^YcH3ESn<77U)(+GGxtl z>0hXAH(yq_L7&68W8$zrh(1O`hyMM3^FPWrm zGWs}V&T|{}VVbTOpFoT2t^A*@4`}0FB#b4fx}={&RzDmXR*ny6sZLi2?ax1o@@9WOzAS`_?%_aaJlL;)-XEILg4X2&^5}s2Z$SMwU{82|I3m2tdhH_{@)$k6sXwH) z^@kJmuI2rq6Q}4g&(ivM!1@&F7l*Vsw7IWkHfdxhlWwyV=xZIsQ7d9|9_qzCer)9G7u({QxTorGo8$SSvmev zL6}KTD~~DTWV7jA+UU-VIVyT7ax3KXn-Y?N?G+ z&VI|>uiq`eA}ql&tiURCJeCLVD|?HTK{E5YaD^S6J(8t24Oa%0^A_c>d5CzuuMA$Y zHi$c#pO41l)(EHf_x7)l>+nhYVLkUoY{qt!p&Wgm(|01PK4eRf&-VWvpby$7N1ZWF zkM=Ca)dxrD$1phlCS8N+Jm`!3hXGG)Q z?CUAo-$~B0^PBSb?EgFcoIjy3-p+jogX4dH^zQGnuODjLwfkLUH+N>~)3!aDmu+0l zKA*tGp2I#DZtGaKHPYAF=U40>IO^OJ;XS}3JVC6<{(%nr2k6n5e02U#-b(#2H27U~ z_CSJvbQZw~el6lvM25rg{zxVv|lr`Tftf132q@{AJc z7r()E`SU#k(f+^5elr!*F%z>f7p0ho_7DG+^dB@vaM=3C{eRm3C$7!n+CsMWOFx`qx0ou_YCgup|7P^ zeo4LhEq#+I`hSyr^Ag{Ff;@oe{LyvpS&zZ}f8mADtNrNvz`1mv(vI-&`=NPh{pZc} zo?jbNBDbRq<*39yRGrQX2gqvj5P1YK`Z4kZ>P{-(3HiQS`PMEsDErYF8vo7lKlbaX zC@jR0L^IOp#Q)#%zrp=KSHuk;@Bh(vADsW0<~@1F8Jx#OT*g&g!ws~1j~)0I{cHX0 z+%5fo{rslY`u}8$@LFdnn{)L4>21EpPTyl!wegjN{=RhpH-&i{chIM8&wkGNgZBU3 z>T|Kuj4@xMor`vrMQ{rQBx=gZC)_zP~l1}&%VCPzJAP>e#pK) z%)Vy3r_q*rF|@PmJKm80(1mVfkVOtvs|){W|Lb^r)IBq{*Zy~72-E3xN7=py?Gd2Q zMngZ__dXl=F9wE^seGc9wC0NyZ5l z8Yh_lBKwp5&wm+KU=?ChJbwcFpI#>}@eKQ)z77pNe|o-W*Bjqh&u=3(V=K0!4CScA zJ{-Uy9Kqmx_!Zy#y8SNPHLn~0TWV|ub#EG**X|y5QlhQ1l?02=7+VC_zJyIX_ zp>~_zRia&;O^l)xk6AUCMd1c3?JrE^_0Q zKRnydQOd99tMnKQWhh4__C>N?dHSL6Q)v8utnvR1#{adO z_1et@nM4Eoy>XiH|Fg#b^$-5V{yNWScdbMJq4Ry~1MuwJ{_JsM0Ehj?^#h)F2uE-X z(cZk&73cNPyCytcTX2dVLvD&Xkza=Fo+V!VsRD|Gq!y z>yT41J#y;<%p{|>W;WSt{mdR>G}T~g|!)5(W1WEPNuFKKhqy5qnD!+ zozfERW$xB~XXbcs-fIrA{r2CuegM_G^?k@_PW}ja49QLM5$ZR{S8qh+Q$GVITswsr zdMfns$lgEKw~O4~d2{4uV`P6fKBVcL=re{~@keV5e*bdFM((dX7tWC9a2^-&EFWE_ zM|+|xzhphaFJ2DUxNqPlZsQK_p~f6>ZGrvw_R-go^+W8xA8!Bs9Q*Ha)jbdJ2v0D8 zyw&PVj6nN)_TOKz|NfHw_s-t$e(e1HyUyP~>im5)PO|@gn*H~}YO&wGm7emfv}d(h zPtdaal~5#{Q5b`9NbhwIis@09ojtFFiS)_Hy(esb8GhZKGnHS}=~u#ZawcLIUJ0|w zxu~YctEDUPN+{)?ha|mWzjW40r)xFl?`si%Z2lkh^aPS6zMF6T$@RY#-^>dOgs}*F zjIUI_>-@V*`hV{$V?FxU_!sMc7c2kEeb3d(KRLMn`@a4k2G{?ZgI^-NWmth#NWW!` zz#Ha#>0Jkfy;onJz7B)yf4MWeJd0c(`THhS)&gv#S5L84g4~LjwF2A8GSrD<{Iopy zp1epl>`=#%<*rp?9}eIUj^G$h;1ptrBZ+2oA{(L4(DPe!)aLwqfA>m=_T5Kgo4XLr zi=1)Kv;BYP=;v_}m(lkbd5gS;8@L(y3FkI>2lwy*(fa>K$$v|2WAx!}N*B>9b_~it@Q0^ z7|rfNVgfsCim?~AZ!Oz5zMTF0Ci`~>+mamIC-;Un!Eee?j!F#nU-cWK^KbWY58A(M z;LgW?vj5q=nd6_Ze|y;fXhusv`fx|(g5EVr z*r?PG+hdF{XIvsX!86EiaX;mrXbd4n#t}Oy?qoBnjUzleS0K&ZiR7E?`VG>#Njk|v z`@h6}d!--s!W#VlBYFd}esc!ra2^+N8CP))?fcpP(j3igbXAK}M%*^}{utbU@4Yl# zkp3vGXnjxGkak~Nir)}(zrwkR+qi@DSoe%%^U(*-9}vF>^hd}QiW4&E#uI*3^Mx0c z4|;U=Lf-!&ZyN8dUM^kRq-(L?sIP|cPcD=$L}xD~u+u#IuJ^26yKb)ig0nZ>6P@7` zqZiRDeegX$);^z&>|^b-vXdCEeI^^Yd*1!Gq4(3q2wtLp!Pvp4oyqBUV=xZAfA8!M zvd8{{iR5JTea?NqofM|hqqDdZOP`MQHM}W54pAP5%d;yzZ@E6jM}=X!`(|P`vVSNH zbIDT7!{Gl2=r3g7Fy@E$yT;?$70=Eai~b8Wb%k9heQjhTdwFm?p1XxRI^S!7aH9X= zSVUHQ&6)b-GOWPK5A(uLM;oWH-r$t<_8CK6#eWUfVjb3F_ZKFI=q!Sb^wd$$qi?2H z{@i(sKlZ{OzX?BnYm_r8M};F7{!RGn<-ZN(um3h|71lBA**`v19{hCJZGNxf z@YwJ-BVYC(eJ_U-uARaUc8?ARj+*U%N1OXHpO`2NN!OZ@Mklg(_rCve+)xkGzr?Ww!;*>Fyn z=W!92aTV9FGwN$c{mJ)@iE`JBG`?Y6BW}$=-E!j_#tIUfjBji)zOmuOaML|^@bv#j zoH<2*fJbKcaQ5u^sWM97{*ve(Z?WX>?J$I*u`jL7#PQ|>fno^n4E}M>cudb zoQmpQ(sf+AUbl{ddnS_erAr#?mr5sUb~*nManzw62_&2RSH($ZV*1TolwuwhpuEF7 z)VJP=_F2x~EHPG~zUVqEfA5#SQ}Va=E=M+QP~PP8=K0z`?NF?5XQ^+*>!1FA#C2_SG)B2q7~4^X za{P)NUPNt~NcOFvqhq~G(?-c4IAdqEhz z6-D6)+4B$9U63bm3NciC#@aVBiDsnH_mTWYX3_G{8H42D{KNP2*#-F_ngdNq?|J@R zJH%~^v)SmEk(=Y&NN;BO)4Hf?RzX<7{~WI38e#|SDV7qkE8v)Na(-Ct+JBZnuh4d8ZE z2HsExFakvwg~9%bHZuN}b`15}v%&sJ)Q%fNXuS2O^@n8iKLD*C8aqIGrS*r#C)(Nh z9q2?CqO}JZafsF)MDr6>r;UMl&NvifB4WlLsxLYJhh8__vps(*eL5Pnxk)5YuZ{lX z`DZg-&;EqXj@k6NsI(`3&o8Xsr_aLzEW#2jLyh*jc7ta~&%ZYQu);m7um)?f4(qWI zo6){p|8uqeC)xEL`}ckJFS7L9Y4&du`=f*Xv6uaU)=kC+klv#Ixq+WK`mMs*jxv-Z zt^G}*lKw3J?4ut*RHn0t+OTe9*+kKribJj)K}=acMxH=5Jw8LZlIKowSA3w~pgct9 z>er7|4w3zydfl~W|K}O)NloalC#UIOkb~`AzxrBQs`OQoQMhCb&GgD|dcR*&SI|3= zMX&L?Gi1-tjNg!b-%u9Fi@1!dxP}|JiQey<*CKmf%@23Tdw76HsQ8TWb#efCYvoV$ zedoC_f-FMIG<}pWmxNLDaVSQbUkVfHUGFJ>%Kc>eROGm`+!?a_l5xf8{GXp&+hhM< zbpFrB>zOV&tKa=IF&o*Ry&C3{rRaOvT!DF+dGx{b=a+;9^p;KfFl1`1?>pRH5c)E# zK<5zo1Kr3Vi)ejP&F<1rn<`btOVy91>c`TMAd{y{LqmUQSmn2CuommE9viV4ThYGs zHSOPP+P~Mdf3KcZYEIN`Hsv;D|7e;RH@0hB%UFM*ADe zA3D*6Ze)-}4viP2|BCb@I_seIr1aNHzjx7wD(^cjoK9qM2C?C#&R;AI=je4K#f|$s z{URDRh?{sOHi_$y(r}sIRb0aj+{A6%!99FDUR@Bv1MaMTZUvsu2k_74zm&Q9S<0U> zm?Rsx^ZrkH9wShMQ5b`9C`S8-`rGK-tAC4becueS^c)(O8-H7^tdcDsntOk&KaMoL zt%u)ghm#{hyT4PL~iRiz>vdEx&mvjHaOQCA4wFa)u zMNHVG(Te+*Pfm~xIzBgVQel~=ke>3s~ zncz;64Lh74P@g||hVUWJI)Y<3fm4W~Mtfe1_!R9AqB-U0Y}vv2f9`eei6e<-q|u2i z&S22~J;?sW$M&yw{#pC}@%g`Z*}DDgUov%F|KQ4>>|gf!IpLhgMO;R@!#z#b%+tH{ zB|34H{z!h!jb#&w*P#6`o>954x~1NHU7r*&&$!{z@chBIKGDC}CjHzExGkJJxQ7RLgeMq4-nXRxP33=s@=tc{Q2xmbca|KS ze;UjFNAoV}M=MfDqYcr22M!a?2ozxy(vRJ9-O1A4<{Mi8{>Yo{QuV=|a?ZNnr5*Z!T%4Q8(>hmj6<}nTP1?gGJ;L zEW-+{!apnjyOqh~%H}R*6uH!YyZrCmVEmz4`AMk1m)k!p%v6*50c+_M<{{RRox<%p zEd2e#pCNqY5S9J)uJ8WtTVW%)8L@uz5ZFqudtV)J$y`CS|K0p>!wz*DqCLj-r|oBS zPZ`QliM~c-6Th&ghkgLPZyVcSi}bYHhvPi9L;Q~57*606VyLi>Ap7;gFgQLoOL|Jo z(eh8C8H4}-_YLC!^eoQc9M0n+F5@b$;RbG^kG)?}Uugf2e8}Cae{hGqhX;6sXwCN% zasYX6s;d#5EgYROH-bJm|2V;(0QxB89_kw+Gf7^Zrv0C0efcbN+$En_|G(S3A{siZ z(--a-zZ-{Q>>R6{A(~s5$lbou{5}Tf*|)Jt-(s60JJviq8mq1UueJVPSS?M~|8HX3 zB08VH4b|q>v(|o36;_PBJei!1s)RXzGWsuq=syV>5^QoT5TDun>-fbZ`?>MwFO9`0ZaU58CGBw z)?h8xVLjTtr;h#XF|zA8`=6eP(v)KVZ;R|d=|?kK&PxAjdkc_88>$``g^j}5jIG#? z*huRS#ukM#dbB4aj&gb>8VbY>3HL;27!KyQ(Z0uc~Ph@`=X%?BDs?|HayRvURogpGk!D^^Aj0`x-T4LX>0z%`{KTBbubpemo&6|3j3WD<4`IyUb>l8%@3-vh(B|*? zwspwdEg5|oGL>N8SL^fBr(!xfU)MgMTiMJk_HD@AW8?p>&Bk1mVjdP?5&Aw5kMHD% zo?pv{+`TV5=aO83Rak?n8OA`!ia)mwm|Tz86#IWC{v-O2*E-bG6NuKBMQ8O6^PBPuPuB`WYX-J*m!b00PuB;uUo!svzBZ~y`;YE} z+NR^$sKeI3?X~`Gq4p2|?EiP~YyZ%O#vb-NnuoLB(K?p>jx@augXaqrvfuf2p_^X@ z&(8NP7ndWxZ6&!62XH9z!!gv{w|;>=A7|&+T{rLl-Y51ip7hKq<{8{~3NgfyUakF| zpbtRrnlEi=j{K3!NEJE$AtoI>u)NIv#nQI7;_ z{I<5q+}uLz59EpZ)xHgxlt&uKtA6vz^#RwoZ{Q}fUmFtIcbWg*Z~l9?`R}9b|Gnz= zj5z9;(6oI9R% z4;8LIAk(*$Pj$m1`rx^luiH;UA3&1cuu%D0s(c~u|MGs6g<<3f#7>%5AV(ouA5>fG z42F~TK8kN#yc5^$fkeZ7eYJk+_}t53jC;qS7!xrWQ!yPKJB&dhI(xel(b?PG$kC(! zI%_)Y9e5`#i@gJSYPENOHg4yAg_**bjkzerJS@N>EWs!3r)Au$um*en;GMlaUcVqe zWPkax|L++V*79G6_1K8b*oy5aLpdt3579Y@2gpM>f@6`Nxrh_wDa0bTFymyjKQKu~ z=l?ZFZuNRPa(`=B*g5h~?eP?8?Dr1%KU-gtrJq5?cgD*f>Yt+KgmH{&RHJ|j3MfVe1yoSX z4cx#D+`u(sm=5Eb8WM;hffyPX&+g2g*?WHOJ-_z^6won>(E$}yP(c}{puh=CK>;05 znB33W80w_w-v05`^So=n@7nKr-*^3Zp7o2~G1$0(VK0Q6j&CFHxJP)`D1G?C8?G52 zaLf1rve|fm7BYLo_<%9S2cYdm;|q)%=$IoOvTKCv#u)=3jvjH{73Y0i_w79-2QcVc z+CdnKYU2`WP#6c${RjIC9DX5GE+`Jeoi`GrF&0T-RVb#{N_*<%;xK_e3H8D=j?=>H zJfn}-zmxqw71L4P;u)joSBWs&afPuOZ<*6o-eoOt$8(Wyk^graf3#ozM=Y!B)JJJ# z==CSf4|m>tEW~0g#WJkGDiqeAKQ8~H6J7Yd_5Z!^Ml`+1*N2ul+J+N-KT{r$KkZz{ zKZP~oT8pRa^IztlLYz0hUie0AM!9b>w?+Qnpzq3YN?jAzm@1{0p|Ef0PVI<;+7YM_ zR_Xj*$n^{f>48^401t$9bGUeE2%YU^4%0 zHZ6G-*U|qe{=cH)aFc!;cX1yNF@Qm@`~EQ$+vg}BhG-vtV2r%u$_M%}wabo9`=jvF z59PmCH-yH-8=+~)8=-mpw$PIJUdWF6UYIGKBu2U>|9S14XW7s1*k94{SQKLdCgG^^ z)ZEZexh;hE;~sXS%$dITZ1{)7Cqwmk>rI~gBp>>BL+z07@>zX1q{nRvnJwSphxtyZ zzNYn_qqQxulbI}aA|Z4`wS^i}8? zrR_0G8)TZc$d010Mp)&XqOg`+kK}`*u#w!1DthXCk#*y^|xnQg{RF47J;X`bCC z;}2hdyl-A=i}uMy?SgyS1=3VlfAE^}{R8h0(cUU^eZ_b5Co4yG(f6QjoAeKs{$bKj zc8!yMGL8}GAsY^Be{E84IIo#(S)^^VT>6(t{{mrmimcHoKjILMAh*pNJshKVHk#Y_ zn)#OW6nYNwZS6P5@2I(cHQtl3%4|_+B6CR2az2?y6+Jc1d%ZU}oN#;!nX}&OW$*cx z_Z;7SyFNJWfZ7k_Q8KMesKb->KTf;m49?*KF5wEU;yMcB|JnG__gGl}*MqJ^CEt+`fvxQ;HmlpR)fOUtyoV%sxd6J-dK?x`BO)wjJzSbf6Pm z=tdqrxbGRw75_ZUM}^~sX)#8cd~7#cm|_FVf^bd z*Cd5i?Kl3OUfaw5H3okbeGTewxK>#Dp1819So2Qf|G%Ytdfq#wuNSrv*}ZK4-NyeL z>%ZCYR^+GgOW?`)|7v!0Vf}+w-QO$XTrTdH{RiqdvB6zah6?P$9_+&b975Ys&j%gD zJm0~d?>Nsl;rXI4|9hC{`+?_s(epj;`8ImK$klkhuB$vgJRA|vF(i?SQdpI9GxS>5 zr{eYW9O6C2y&2O9_v*I}VR@XuDHPfxGkj0V{nL)m;2g?*GkH{eQ9I;+hHy#P6;vPB z20ttBp;o)Tu>aKzJ~H{Dp1kUu>$r*AxQqLEhyk?uw%g^unD;u-wfC`YUf6$=-oVap zT*CfeF0YY=^||gm&U>w7XQOLtqjp}#&P{3_6M21W^A*tN^OowPsM}qiTeH1F~%VN8g`su)jzf z?OkJ?g_of3pX@6{#_|92$%R;qrC5fFZ)&$#zkdZit_8A+%x;taHkn&RUyJqV(AMrm z*8|_jS?>Zp*yy)P`EWD270GVETvz(++v#*>E5p|K-qdh&+P4GNk{D!@@Co66IfVTxWdBEcF8l{^=U^gSzwS{`5z! zpL0X~f{qW=FDKP6m(?%F)i3CoW8Xb-ecV1xInJO77knSp>WdnrUgd{I-D~PMJQ;^p zomT&4+2`HrKeEno`m*u-+AH~JNOsSq@HbF&A*`j&8`$L;`o5(8EkX$oWliNLc#u@sLe%xV*g*IUq@_H ze*mO<|QAjCCOIBHI7=$%m*~`EU0Bfa5{m_G})p|9d~O|LU0uRzZ%?{J3vy+r;-yg~ce?QTjLwc<=rm}qb z^sT6WplrG)Z||4Kk^h-JM1_~30v|8`b~)aIeet*dCOyIYa{3V*LlP-u(1aX1kjDwc zeE`aT{YHrP_9@4GU;LwRnmmJZxPa!@&9f!1;3}>ow~~z|{3g9H|6;sxWAwXtI{(Le zkN)@j!X9D(u^bq*&UZs5)5?gizY!|7yKnc9y5;-0=KJXNecWIlDm&^i)cL*agz};{ zL*M7LAO7)~Fg%8Ru_%lr``g|^+Lp?!a;Hcn~itSaTx zDpfv}DxXS~Po<$@b!li^R~nkOl!oRFrJ?1uQoi-lkaJ#Zwlq{PDGfC%OO4?u4YjC4 z8X44Mg8w-nZIdGOXD}I4F&)+7tU(I3sN4Id@BdBy_c!_9-wZRIGaDtCi+Pxjg;c`QP8(MjVvq86Wx+dC>eH6RL|HVC+;xkTq9vhvP zH%7f``M;%Wv#_nmyzIH4@SL1mO|MzxJbLXm;huNe^Ugf*e(xyv<*B#6q)u>c87i;~ zd$13`xBd(};opt_+0Xtz%>E~v7qI`A_%@fTpUAlWO!MV!;edD!;Ruc)cWYb7B1!Lj zjelT~_5nSE!upqcwSN!t54@~REv)k3wvZz`kgVAj^5h9r(Nm+eE5_OX*70d%&THFW z*3KBLtxuj2b`BSC2~Wpo&I;j*5fsPgS|kZudo|9ml!Jv7U|l?@n^wWK6|$ z%*1Szplz7`-x2(7csjrOyz4Hj-^m8$KqH#Ac-9*{|JOW!?G8Tz|?zx3WuDHr|`|5VDJcr&bxVP7i^ z>&d>)m4@q{;Zk}nTQP;gxHht0y%_h3sZeIV^?Bv=PV4PBuZrAEZbgzF_wOmCm!Sf? z@MQm3K=vZ zhqfE)e{?)h|D%iEjr?-;zxw@Y`zFEG9jARGZCSQ)-0QIQqVa!?%Eyn{A06UM))@bH zR2=l$N7^PhK`(#bH}QeC$#HEIvhKWde&HQT$0@&`My^JCf;@+~4q;(li1NR%{@1g< z3woS47U$k|<7xU|lYX{&oi;-pmypTI|26u5^Z{KE*CkxRRb0nS+{S-6{^+9b@2Kw& z&1gYkjq$y{KUA{E?~3O>9%2AVwt1C4ph5qq^skmT~TA$r~!Du1yR z(52n-w0}dpXR+V=zMBY3$>={@My|jrtif8W$3|?%R>ZZ;OUW`+FOdIZ`G(pWY2uHF zdlYBr^(eRge}(gQVGs7<01hGl-C?0^o%}yo9#sxJwf`LF9rut87xjOl$$8DIJ^yW< zKbb@866^mPi`Dy=Z2p$$1N_9}@qa_uZ_2+T{^J-@$e;>bO|-+iUeouEC+qVy4;db= z`=6V*jl0OrU?UIaTcCGtaxEUx2hg*G?*ZMbACI*k^d0rj0{aw_Ly_DtJPapCqUvS$ zJ$vd#o!2~Nc$grbNtleO$h|E6 z(m0*osSN0FJd-{fJ&yD8a5vf2Xk4+d%Gb>wC+8tKczBpkE<}}drX1Hw!(zuvk-6sG zJFdH@9ZCM%{CjpZe^%XLZEG@v`mA=lYgS+t)?h8xV@-gw>4 z1pDH3>+1+Rgq|VZv#_WAm#eL@Bm5YWNFgUIyGa`9P4O5VsC--AA*&9W|3p7QFV`=6 zicIZPKHO41JcxNn`9QC~W?Zq~&LIEigG2e(28VO>3%GoK8HM$k|9Ws3N*|7q7>)inhKK%Y-_IuBqT^yrz$E0R`Nl>i z!en~qINuL_Dt$V7dfA4Ll%eRteg8F6_-vG*!tq>k9_FKZxB6NAmO89{x~6`5q<%W9 zenLGO8u<^}`43)YOQYpA_Vp|5YsCF^pYH!NOZ|+({y#6OH^-=-5!e4;=zkVtDV8C* z!TL8#>?=pF)eeYr6IapmKlM$jmowdu`|#_??TfU#U-SL=4x82YJ2^nI+x-6~;~`vA z5uR5*dInSc|3v>kNBe-xBl_~I_evj9s70N$q>)jN*JG_X)?*_!fzy?_KjgTy=>Oj@HaT%_d}@IusOVSaC0ahvNh~nw>cc*fBA`e<%r*oA&Fk|RNs3| z|JovG^B+0e|faIVb6pg;oWUR!~PlnDb(y99jbSZ4z-C-DXT|^^oEh4 zE;};Rw@(e3#23QRZStCHD~}e3V>S9^8^?zC@4XQI;o6^uB)$9>Z1k@^ADWz#LkIFW zfm3KSFRbC>)X=PtvFXO-ki9o4v?M+k%D*!%oc4QRelHs?`mkd9&j>q*x7zeweq~&^ zM8AToxQ?5+jk~yyhZw-1jq)q{wwsSk_WsnEJF?%{!I9)>j72dfU=k){D&qcr)5$*m z|C#Zak9KxEuG9ac9T0uv@&5Xs*LN&@F7jDq|lZ&J|OFV^zC<&U5>k#7$306|18kIDUL=op&2bz z#uwBWU!d>5b-(`q7p0c@Ro1`B( z=QJNI4jp73Ipx3!@)WX-!e3P`;I!j2$j{L(P=<8P@(z?E)vpwXnmEP+@xSV}iTj}Y zc-w#N(EhIr;hgI(;1aIjDz4)u+S1w|+7un@{0~{!f11oY?jajK(EdkLjc=t&`?Fi! za8&yPt^181di{C*7te=|c(F-o<@9!~h2UU-AluVmL-3+W3`=#)Z-Jsx9L}a{r(( zmR^hrcsf7-j{5hJGU1;3*S9~}Z@s@!zma`!Bno?R?ARo~Y%{h;SYiEv_lv?z`fQY7 zF6N>C9et0lYX2j(T>Y*btJ@-eWV*$l)%N)5;4t603$YmauA;D%T!s~Bo8tLDUPFU~bs zi}l!u&De@k+=%V|-`G##Yrb2@&HI&8hDcC&O{v7tB>o*H2Z)1+r|c>?4bHz{n_N4 zC;bOGdI$1RE>Qn!6a3-)zv`FOzY8AM@&9i9fz{816Rtgl(>Q~3xPVKzg0|P#|LAyK z{OH=Een)=wE-v@nHoAJk>neZxtNFf$j$QoOtX#;z4I0I2Nu(pqUW$SGP==)oo|@Kq^?=! zw-rckoG8YXASQ^8W_;AFb&7dF-##*Z)1f`!4%U zi?jSQd=72w`meN$tUBFhto%5G~zq# zT<)Eb-P!|rGLApq?E0#0=1XrH7OvCBI=2`TP`S>0@crh4A2uJHtbft?eBqOka$LLH zn1gH9>2o|CGcg-EVcC1bLkYd}DEs{)`;HZ5tq#rH4>VM?Ws(z|W z8=E1X44RNb^1Su;*%2M|+DCrB=R2gIK>b8{1L@1Y4QyX7|0)B^R6qx`Q_&CoFNPQZ}I7M3GYV!qP~7q@3j8cetmqX9v=DK)p z;x@|lf8HhU<00~27#s%3L7U|<^nF$Pp6va+xdCLQJT;OWjjg8I%cB;bJ1^ahI!=tsGENf#~duA z7v}#zJ2)(+=Y&Nc(^7ipEq%jSMqhy*&n2(C?dAvQA`A5o`?9d_YPWiwom$vmhpZp- zi7?YOtFQ)Zu^t<-8C%iD&Td!Vb_`Sh4p#q;Q~#2A^vvLYSI0J@iA~<@+!pp?*7&y^ zT2VPg|ABbQP=Q@Y3XA*N?V-m#hf>%_KY;rE?Ef0}KR-Yn`SJ4Mkl&Bs7|Ory8~ZE$ zar6`_ex)A%n(vCQ~3xPVKzf~&ZWe*H{0$=hhYWo;hv zJ|1EKao>Y%uXT9-Z_j_XJ^(yJABvur_3Vz}!^qWlZuFa~fez3q|wFV3g)OX!95r_?Ql^@oP=KcGooZ$7F1*DlSHagV50 zvf`&-331)^VsTX((>H;vN*nKIjNT;rWaK|@jy@klE#fzu9_Je->xZdZUp79#Inxo_ zoHNPhqsGsWCCI-b?=FxpH_7*82j5%gI{tpY&BJ^w#3^}fF}W1WumY=4t)8k`^0D^0 zx+^VDXBw3c{D=+8ipIy71n8NrZ=XNSz3i}U}dBpvPBQF0o-{rfB zaKLe&@%@L$BdGX>=l+*-!ZCUhDU_Gn^PFr#4*mKD^56bS=vB{mIPUvv$K(l|!f7<0 zHD;JRhYPrb++Ock-F}7M+3nlFRr+=GY!U~a>>pX(Jtx$dPn;T465?F*x@jdLy{IH) zP`|*ud)It?e9ukCw{aKu(PmzI`{k0*L3Y0Sl5hGY{(bZ7={+yMr2ozQWi*Mq87+6r zUq;US_SRe4j(hECE}jbI>03WD=8~Q~TVnn1k}&9dvHo#yc!oX{^)uWX;u-~Y7%r^& z@JnGNIU2dDm%>=G7}@koq2u+JLZ@{GCODpiynE}h7C_fJ_qqC|Fj?4COvg<0ebYJ% zWN(-gO341-&IxnLd6vi(iYWZuMd`{*a_mB-)_OCdbcCdfZvO)et4y}mu|Bi^~ z7?Ozl0OgjjKV}%;=UeF1Chyqb{`{6foKqb4)_AghO_Q)V|1U>&ASrL=$rGrG*NLZg z4j;bb)5zRWZlKP2X;jxpKT_?^LmfSh%q;H;+m(|~%HK1tKZm@1e=d<%&~`)liw?XOqaUrX2`%ay<6AFlt{|7m-?iKj6C;hzVG>-6$(s9(tbpO=K& zWG{J_?E9iM3X~!DW0-csLo&B*P#7QwZSlQt(oah^uVx33!%^@1Rm%s6BERMcnHP3n`lS9={fUD!;_MhQON zzcJTw{@eO=$wL3P{+#*rg;2by=<<3K!H zQHnC;q%qFXub_8cQ}!O@f2Z$3&n@E@(0#-Bg?q*|2s?m7ID%tHB8C1R6om}g`_JlM zvhUaYB`>NkqCKO(E6-mQ_v!8MTcP}8f8^=O8QMRd?+JSJiKO%qoT8sbT)Qx?VNme@ zZ!*?EShMo|9C-n8UCB%26=dm!b%A_)R~=tR-m~dJVIC@3Esxa1e1clkWqtqhYhit? zH2>GHpA9!%cN=$c9}h8rL0dh4eQ)jP(C^lXr|o-s?2BtSFaKYb|Hq{X#I^j_im&{~&xZBnMr_7bl%fn3h`!D3=>M!a z%j7O$d$14rpFJCPw%9N1Q}#R4_U#L!LoYwnA;0x~!~ACZ3m%~#Lq+&xc#Dsr`S7qX z$#D`XWRPS3WwXOV6TR~RJMfM@-sm0Z8N;^LuiWjNF64z(UbKcEc?!wB!@_Cu460n0 z5>IWnJ>DE&K!#0Q&$g{|UK-W(nw9Kd)S?bg_qU^8a?KT7#dX}oZQR9uv@Njyz#`-P z(TT1N#up%u9yA>PgZ1|uXOTlInr}Q89*QTwJrM@T1IYC{NBE%sNBRe|;W3o{40`B! zbn{zup|Jn%%ihPg>|>xWvf^Kjum4AVKh7PA(J23pK3cLEy}#7oPxk$V=T26>{#=+$ zPQ`T0L~%k8TtNgYGasK>Tay{bs_`>{o^~pxZaSVAJ`xD0i)hR2| z`1_;Jh9g^shQqH94ey=(WcW$r%y4Ma%y97Rm%@*Gr-vV{n;s6Vp02O&rSR^`lJLU~ zC1Kx;{~Gpgm>qtwWp>!UUwL>~d3aR+zyIj}bM58tJ{L;qeb3wX&a*6|_x{rN`&Y_K z`RSPFef0A1@Y8m_oEkm?znA~4DCCaP|J8W>e<@O~Tf;+m_XcH>vZjZflBe$xmQ!{` zUqOemsO+-&hl&4cZUme4XUhB$FNL?pyk!32e+@gA&knn^oBnR%e-8UxcL00ZXN2#M z`Y+*~8UH!#-XZ+xe+fUlI3(nMJ|-NB=QIwpw&ru;-7Wtq>|gnx!lA*>g`a4TAMx8U zB*8rq-qTJ>(R=@bKZ%TU`N`mp{aG}ru(L{s8diEQH6iIx%nBhmlP z@NmlSH7E6{QAhy>sK1<=h`zkhYKj!+l}5AE;(*{dvIt+ z$6@=w=_lxFH#ZD2x6HpIn;+T#jh)PU8T7GFJE~5822VDjk;-r!)@m@uVjOh_mLZKO@FR57^X|0=Pwu;e-ibD}OICLe4K?YZVW|Hcj*%FRu_(p_OhRFQ1?`^H6z!i^wSN|9|9C%{M)p4@ zJ7+4UVGbD9qu|47d|FQ@I$ zsvS{TRrGt~ALolRd95f^oh%9q>9wy))9d_7^kt~eO0%#u>hv2`eAD=fPui!>Z_PXT z{>iw9c-%XD4Sg-vh~I9h4FZcv_I;sqwD(3*owIReJNRn z3UtZGF^|W*9ryhDLw{h)UhNG$8E@1;Z$uNCu}d6#unz}t2yxxOBV?aCA^)xC!!ddi z{r{vbMrP2294g;dhD>?fri*<89m0~k&0j#CUOQM?aDsjs_4Ev$`2Q+C>-qhBTsY&m zbGU#@xPpH3ey@_f)*HM|_I>vg;U;+-cX1!h%9e-Z00wRIjU(5njnS^pi5|zK#6IZZ z^pSYF{?9Aw08}orU%YEa`)w?Wk=&sFTR;BJ3Mf&p5 zjFvn4|B*v0s`cmB?DqfCQXAJaAk&UB?fySo9M<}e_1K8b*osn=p#p8LX;16tuPWBR z>6vQVb(4klU9<@s&ia2ep&2dW%yt_$K(@NBd04SJ#y5z4IDnk6EDq5-M~NFp=*Q6G zz2r9(^RX6(E|huxm9KgaOS}g@_#{35p8iDVR?$<7qziAop>6Ut^NMCjmuFiS*Dye} zyjP<>Pp#Iz-lkrEJ+`lXqo_wYUuQ;KO~|1Gd7Qv0{9%6UIxj!+$K}bM54Hc#YX71M z&GBLfy;KiRz2eKi=~~E$Wa)2K5+l&7e~G3bFk&lpKzc7>l;u z+W+!k2Vy&`tK0k6F87Q=57}^y{eO%7|A75}hy8z1-b7CRY|Zlhf5ti!;_3VR3toY?xdv;o9cP>;H{Fbjt0md{;y9m<4Zh1Q%74crk)NhrFvoYg$aji=*Z-|7Nbi;Z zQIFBC8H-{}z$8q@R7}V3uYawry-D8GwvBs)_3#1yN9$kD6vu3opz@KqFl5yY&tROr zR_ODw5UE+pC*{JEb%Qfo+@HL@*l$a*4CTfS6#P#^e1Eg}Vuh{3TC7L@m(PZco_lW&}d|0?i@6>19kr*Cs)9<3kao%w^*~JfVUsxZX#Y3|9x7M;J2W?kg zVkm}VBt~N_iqZegXTt%}6$y|IfDFO0N}13Z?Wi)Q@*>v)tnx>E{Qi5Ej>c z*hTI^?twKb$pgq9_g&V=|H_9$j*p=gQ+zxrO6J6(xFS*PIPxi=H4KJ$y{oW+bX0pZk*-`9+ zan`>lJH%1`p7Y7d9oorc)!XuaBcDCJ;C^s^D*uoA{nkM9+d~Xs&<@`{`t)NBC41|XyJWxmZzMSy1^fA${C7+LZ9M0j zd>mgj4(QALcHbKwiiJ%;{#|uG`DFb6RQhzxME`G%{Wt!9HoaLHUP9hi{?8@nAva7J zGsJoY^v+&3$r$Yc`eO8)l?QI{kt6!bbMAYR@TFLW6jj!;pvs;_hwJl_-^FyP`~%1{{I@bvVMb)_y6zm`yT8=+&?0Z z1N1{E^gp~~P8Izal1QQY@JH#Zl0Ipvlh(rghV#<*p>#T@2|08ij}thB(`Zwsx5s>j zPQ*P`yXD8H_IG?fyFaqOS1bRwc{gMnqtQw>PvOHC&pBMcCFB-qBTTgZ0KL<5=uppJ zrC&$SQE|8XkH%PsYdZ+5)F!x1-bHeh^#{m@D6Bs)N4ka?C+2uinLMn{slTS3>%8Bb;o-`Q`&pY8plC4R(Y?7z1Mr*Gd;WCyLp}80#DX4%Fydk zhcvdgTdROAT*e<4`*JG&^0$fTFUWsM9&ta_tJS}o)W6&0FS2f({GH|B@77O84tLH- zjK)|LV*(~>Ih>)c>e2p%Yyr4N=CCJ>AMIa=zadqWtHN68WF~HnEt#6wA=B9$G=h z{{K~EpY0-CQ4BtGu{xFjqmEPHPtafRNw@^^`jivx+P^zwi4PG0|A zA}_3a)9(^pTYi_Qel(D%nK+P0O&Lhkl67+i5@|ByxSq^^fltu2h4J@%0XONlaToW| zw&Qn+c61#4U7{0RZ;LzYKN^3R=t=)B(Xf0V(Rl55iKbh>OEmZXF402I(sLJomuN+0 z)$bAy#WR3Gf1~_HQdpHZhSF=tyEpL;r;kMai|$QWdWL(QHINuBtog=3Vk}vV-2Q>Y z1acCx!aEuV5}oY>iOG(qqOdQFGN5bkK%yJfx5N#KgKj^rn{OJ`hx{*f@8m39V`)o=KB%0A8&u5hZIkNRe9RK$yu~FDjjj{zH?ELO$!+PhG+f%gs zAB)09dc{}O-(NSDhQ1Z0D2w69^VL_2s|In5T%mox-}t}7@;_Ox+}P#3J=li>C^uI2 z5P1a0(8mAYjyMOf6J31z-N@5>&>&woDg&BM^7p0nKeqD+B1dnnvX6*3QplhQIdq`^ zSK=Uh^`D&}`@XGhL2lP>-k}dSj)^(rxcPzoj>(E|>G%07ed+Yva%~)3qIbTk++eF* zp5p2<-Sed=riO{3`hUJ=*K5V zqf(hRmaIB&zcq3KlJrUBWYo&EgqW7QWYw`E}Wl`w=+amj|7|$Q= zCw9VA*Tr!w)5)2bjS}>J&AI<#&C~A|sq^haBCIfv`B+bsV~zf)w`&_XWx6*EW-+{!Wyi_dTd17CieF>|3P+s zsC{u>y`?_OlRb_b)V+=3ZaU2VCJXD|sW)RC+ltC@pU|K4>%?YpZAB@PuNeP7!<+_s z?ILyDtDguJ^j#>7Kiscwds!VP{KzX05`SOyAo1SbM~TA=9wZJed6f9c@<)ju55AK) zs4O^8bvN;&m3I>FuDh4mKjU6vU+;~?4|m>A{9wti6ML86NHjK%3k_A{Li5{WLetCU z)+9a`TJDVrd;Hfv96(u(Jf?3ejx{@`-=T8wpb+<@JKAjy;z@0L=Tx%ej;YtHthxC9 zed76$*)+m9g#N_eyx5;OB93FI z=)ISCYsCFT`K+6Xox5))c5S(z_`BXaiMak;%I|v~JW70Dnext@yNTT^?3LeCuiG#m8O zpb5=rS)?BaIkYYi_DWH>DxT}OiQC9+D+*cRcj=wkBEFrXaG(AVJ^PFJAB)&d+DI4> z7Wb|jv{T+hauy#UIUH5OQ+J9&Ek-(ydl_cV7KM7$^%jLRdd>MOe^&qh=Y6yM7h|2* z&z>oa|Np|UQ0%x*`(Xmv`&(mr<<&{?od2e8h@6V)hmr6d@MwAn>9RzFQ(VN?Y-}KHY}wtL%rjS<1|^fm%nAy zb79B0=R(94X?7l(exVo8!hxKa?WeL z_gtu4=ibE=*T&mL?m<%DdzJ8g^xEBg11PjVzOC&{&mfID9B@vv`#(e;K^*^oj7%ci z%P)|X?(W!!uZ}@p-5=-WJstl);-mQw!wKm66s$q{d@-H)z+B z`2VV5;R?Mr`UZ>-xJti{`tj-wb=TAJ|HIUWe7&(gyy^GbxQn=Ve8n5`%CC#UL&pOc zR3VRGD28Js3iC4DV__Z2iQbF!w;r}nBAIpE(e1r@&TUoRGd(}WdmZIHd&k`qp9`bK zF&4#`fJvB)shEybmFMamRG;*mFMF56-|#W9qX%FFxS%^K%#MJbZ6>3>%~meFgKkttNr zccFfVviYDgdatsY+#@XZA?za$AV)t$9zpCw=uoG}b$O3DP9pEP$8lFy8-T37#{Ry= zzd+VLQvZ`r{R_*5yC#DsZd@~&jd>0dqzjJ&Yd3w(j-^n!J2^!M;!zcNNkMjkfg`WLTJa^2HLg|qsF&+Gp_tDpF!erj=C!WCS_b=<^l+(p|; z{qN{NC%Vv$JbKX3t^eD9H0{>cO|~4?w@v08x020cio<>JJj4J7{jK-D#5r?{!!z{G zm-T;dDGo#F!_nh7|C;{qMf$(Vk-{pM7l+a0SR_a2PbDWHj$ufNr*^P$1n+C_@IPh7 zNtfd~GCfNFw=~yCa|*Q|2)`&C8G8Nk;@?~UXOcK8tZnhucde~LpNju*{hwF+aUiS46o+Nx z3dCnI&of;`uRX4h`vdQtjC%~kxy5mg@fyFa#d>VSW|V)%+WurI%20t_*n@pIgd@m* z=dn#vnE$fHT$fN}{GEE{yVjWZd;ha9ge2Kp=lPO-|LVDtO~|1Gd7Qv0oJRlG%@-uk z;R5>JXGfE9{hur3Rb0nS+(z6}=q`C54>5q8u16}jep$k&Hq%p+;Pu~z5`+1 zGqi(e>HnK0|0A_q{@*G8t3T6Z2GK??tp9Y+7(@(reT6aIZxsy=Bk7~j=KZ!S<2%UC z<=)AwAJzZzcn=Em|EtvhY5D)C`k!8ypX+~Gw;2~Ij$%x}BuvIsOh^CUn*Xi+)cf_~ zFw=40*X<=imS8UCq4E{wC%F)du@p&RRm!zx^jdj5^{Rc<=&MlwqO=K1&yZH>F6^(X zZb-3(YuBkC_?ip*>(c92o44kgHCT)F_;~-%M#pW}*xxtQ^AFVX=%UB5@o|hm4|}^| zoch-}O}Es)WXmJx$X*Di7o;@E+Gg*0&pkG2<(?5@H=X6cfPx7*U0^wJYeD#?^TYS5`SlR3O zjk2GPu$b1k=V1orUosa>9*OTdh5Rq9fA8A3j>K&;d(r$?@;-9vzl!a~(|?(7?WDFO zo~*0YGfFhYji5Oe3QXuJ_GW zkCPt<`({xqti1Z!FvfyLYsrdV z#d!zzAEze|8vnP~xJ7y`dpw2B^sR_}sxhr`%wru&g_WTKyRZlQZ~%qt3*9Mn)IV?iMU_!ABb!1 z4|UCOjKpY+MKLB|5(??R_Xp|U%fEphG)QOTHR<6f;t z^jdy_xc>iidY*mpwEsWe>nwf1!sD2qtommrJbeHyOY=l|9@!Vw!NTI$gB%Luf0T2nd(t;7 z_KRvG4VI@6=l>n?AIFeF22IGJ19=qI|C!^N-tug(dB%@CYclV+hisUl{WpVs=bUD; z#qr12|Jg1)j`@rGpZ42pprXP2g@5Ar6n8)0KwUQSlen5ms&%(bIVw-xSc+wcK8(2k-wJx?LG|x`>krb`phvx$f1v)oqy8n= z3ai{?3_ zGQNv?a=S4aJ3jP%oYcPlrtj)|{FLH(tIoHgZ>r6Bj`oYvZ=6RQ>(RB>*beE=@0b3! zrGJ6%Z;9`3x$lpBJpYuR=#KUO(Y$F;*dxAuIDkXQW#yCA=H=2m_dfO^9iu1FGef>o zPIT|!Ln2ec;vSS4vfQ|r#M3G(HTg$7jf)$O+40IKA^l`ER0rI{IkzjB|hJ9`FAA*7v2E{C;fz2Ib;c?DOoLVobm! zlz+io`k$JoPM?bYH~veQPR@+_XV?Q|2`au-B7eOU=F;b3KKkD^cAkvo#bPp!4O~i= z-FV5GOD~1(cU}sW=8Atm@wsqcnwI%(1q%Dm3tvUAHGe#{x+JWjuSNax6783gkT#dA z&TmInemVU8rZ0#0j(;f}&dv&ls%C_rY%=$2%NN7J4b#Gb@l(Q&c1{cLuKPmRzxoU2 z%YHumuxd*9!J^NFz4XRmpAHR!KOLIQOKMv3X}*!qgqG}Q!g~L;5u33Ur6|L(vtKbz zD-nLWX_o%4uY{u?$Opn>AIvd%zG_^N^&yMGKRozkNa_o#Njw{>*XjTIP(KFJXFv70 z|LgpSkh%CN>o!abe=}ub*d>lVs5t(G@YaYap?utnVdtn9!>$*ngujdT?Y9H*oN-?a z-BdiZFtR!sLR^5TJ!PXh{e|dfNy$y51 zC3@dity}PAeb@A>=>N@2;W~K}6+dPNo6p@WZRMXV3b%#b#eL+4%rX8As*cYI!yS)Al0KRoi`qGJ_;Tii^prWF4)vp~ zm4V#r`n&x;0a^MaviVi(0+Le^_Y&$s*KutFbUe^jz;wUOMD-Nw4;gPzSbr$3T*?}M z(9KUo)?eh`P!>Gt-%sFG1>nHzbCm2D-ib|Uq$x5As>?czbXl9$@SQX&8TGS zY$dBU*$;&*Ly}%W?n3P~Wiis1wXg7`|G%1@S);CtW9)0^sFx|Zp9vvC;vcQ9M`7W?;}y0!Ns`~R}A8g}ZG z$Ip2CIOV-KbHd{o`3$}D197Woo9N}n7ZmJ&b$0hG^{=`$?j2q!-V5=&b58Q0^T|A_ zZb(w9>)dl;1zjSU>|GMuQzo%SR82?X?^>ShUU7Bq!o>MrDGdPC} zxP&WcQb-o%dNSp==k)hXPtZRxf0h2Km(~)n}-%PgMVsG!1uXe~+h22 z-W3{N(*{SA^P08CTOP5;$sAhG%U|l{9fD$PRJ^4 z>ZJT!LEnWvNXe_U_`~%P`hISW@=sa6&ssfkp5FEi(t}u@?sHwm&-ewtJ}exfA3?u+ z?hoeT(vwIbgC^wAfjmy2|NXfB<1lNVTmQjv->0l2OP;|wTtM?Z{Y~T*T*YEh?ZKCQtHPh!zjHpae|NBdce8(YvVXI~!vWXa$3qO@c zav1w}$nY?dUOR&QORtFYUyQ-3-{RgjxW@$hSNLe>j72e;v(JSIWY#_8+`}aIJIV26 zbQtU3i7tKnPw&q?R=>^$MM|8th;s_k$epo<$eE zTl(_S_1OO>{dc4v%_rIa$E6=Rw4(BXK40-H#!@Upa)k7cF_wZ}JHx$U6@3lr2fH_< zol`e)`0v$EYyG|+8&R&!mq*1{jZ3p8`c`43C_@EyVGpWz%m4f3f7GhK>XZkuj)-GK z>mS*x*Et7p2uIKpJ{^vcNus|0g4!Cgji&<#Kf-^7L4ybf`B^&`+VTujfJecd!2&Upy;7wtDzhGMx>5J$8+x`K2{Ly27fBsYEm-{{c`J!-_ zypM;7Yj6&bgZ6rsztj%0XTdY{q3AOQW;ogV%jd#Kvj5Y@D3D`Oj0u>8$(V}in2Fga z!CcJ4d_@1>&Z3dV==nw*$29^9>&K`kWBIaJSo1V~ezN=*=Jb)PkbBYdV|%Tk$2Cdf z8W3yg>rq($OB%b6%l~Bc4dbtndSLt&>gZ`?)D88xE}ctVrw=H^{nIwmH)AVG(WVY= zzo?GtRsW-Fue#~5I!gT-_xfpgRsDyiEiv`MU83 zD+h;t^jLSNaDaXYg?)*{o8F-gBaa9>hGu>8Nis{O$P9AVJ`py%|0a58w>bz$jSVn% zuZw@7hs>ioq5hxfKgO#kQHS&t^}n#V&T-DQANOx`IL_lllwV|jdq(Xj?2q_?^m`WF z$kTh!Ag;z~(mzA`he-bz=|_&HUA;{a^I{ z`FHAk18HP@L$R;uzH1(00E700?e9c8mmG@WXuGBTE01?5M>= z{j5#Ga_B&uI~mKqJiT^^=Ql_BNk4^p-$L9wFzzuQ*FdY@J1*25w8sC@aiJDx4m5dKiT?facKL{_<{4r4_r2Wfb6g)l%4 z`l0qSh9Y;%e_%MhbB45yH_m`Q8a*4NO}~1#|LpP~Wv}aNJNiP{{+jP^k@@#EFNCqq zD@GMPwc35TrZA7|*Pgv}$8`d?=KC3N_IM?gx`v2Z{Gg_4SS>(nTe}ZQ3c&>QnVLlcjCoGG_ z^v+?vdn~0dLk~TVZggRVuuAWD6}bjU`dV^5s_3!rxfUB8Z$^e*k06hvt>$c#(&21jmrmu8(UBCh48p z8?i4iMbDt8*Rzq|y6-&p6<5mlP4OCZAdh6b{L?7U(4Uk)r|72<*93@b1E#0RKjax< z=g|MxFN6!^C0xN(T*uS&jp9>u-22Pt!)>zfZ(j&^$- z8vnm34!r??X@7t(*e9T&I1GB%cZ#7Hj*%FR^75h(_ZJ*X&+;$kMkPWqeF7%o>G-jG z(*EfGrR?rwtgi0G->(4%g~L$c2o>s}7%}3naD)_Z@j57`7%^gs5hJD;F=E7@c*PV` zjDrkk7=$BKs6!npR2XN@JncP?d(Y$ETTC%xiYca;Vv6ZF#fTB7aEn*G!nv=t5w5m9 zx4-+xJD<;5`}VY?DY(wp}hW9;1Nn28&3 zBj(^%+=dJL?-m(CwpvyDwMt+_>*?0I{-72eo zmE)UT*dJ*l`;U%=@*mRZMC^a4l}-baXv6vbhcvynlnq}6dKS4#b_W^HsSj_6i~SF8 zl5ZpSKfFV}i+uls^y9k1R9apwU z8^C3)ecJ@5bj`CQvV&qitU2V%WH?*|A+pZ%<3hg8ZJ7>PJ{w+5r> zjjpYMUow_H9?il@;RM;_dI$c%#y0WaUtwc0(J|%L2AD)%g==sfreh`sFW1jX-WYZE zIgYhBzn)`Ue_)RI0ev&Kl5zcwJK}LqYNLKjJHTs2;V$93aWC#ikGfYcx@rnTx492l zb*~)y=oxjhHu3EV^BBg+M@Z8YJ1 z2p+=|cnaB;QQ>K_ZZiKD4QMPm=Tj!8@iF<9ajyMsj(G-aunra2ge};HF8%!7eC-}S zckc@R^(y|gzV$4b<7>C@Z(Dcsf6 zb=A@vkXymG7MEGY*GA0u)yMfovHnM|J?7dS*M=H;gZ6^hhn^PhL6RQ#F*rZZziz4f zmrg^e~lQw`G2$fuXO)A z-K+ckyZv|l$9vLwA0OaDq*uBBqsBbYdl$O@`J=-p^v{s1lJ-{rG0**rKPj%d)jAC1 zm#CdFI-DhkYI*Jk;jFW!R->uaxKUrN~*vgMfkk2ZO|{e=8~SRO~ZTmDbU z`w8zY#r|jXMeeqixHMB_T+44JJ?u%V-+n-ubidypz(Xh>HY{ZE2tDrk^%(gCp2C3o%F|^3J?c2*8mvPF zzS+O9Nq7sk;pbbw7Y1K3|Km@}rr(=iB5vR>qr)z84=OIxuJhusP<8aX;br{ln6(CH z|4Z0;#yX5A{w4fce%vKrZk>8Xc=5pZ!Y`&=5w@voS6UnJrK$fKemUX0_G0_j@IwD3 zVe^zr!j@^5grA=_)=Jvjv1#_D;Xf)b4dur!3+oFn3l&E%3mZ;c8fyL4fORu25C1;y z^6*Oiu#gmPLmEBEV#mQ@;aBESy<~ll%A*&D?f7Nt;_!yw-o&=Wqrxw+dus@93s)CB z8}?M`zw3TByteh(@Sjyb3$^{Fp}uT&sN4Os(75rZp`qZXzS$>3)8e0o=Jij8hmVG~;>Y<0kLqK1G`u6tckv$H#|QWjAK?>xhLfl(G5$b1Km!`J3p5>3 zuh?h&0h)`oyF2Ea{{NT4XEEfL?o)q!7aQ7*o~hbDv>EhG=Kmv$99opYt!P6#^6ek9 zl;b7*e{BQp#TSK<(#o&@Yp%~|dfYoPrEWBq-kXv(#?vPvcS73g5&89(YKDhN;;I*4 z6s{t#L9II1b>wu^jB#I%Z^TUD8<0G}K1eUWjzd4YQP2NpOByo#e|}=q6zzWtwaK9w zH#%nyZpCdVzsVZTLU-E6L78i$Zyo!t}G1=;s^Cz4PM5+)7y|n589V_ z7Pmhevh+9bCgQlJxEA=^^j`NL*JgQ#{x0I46v2xKhFQ3Qg=A>To~(r z#$zHTp?2~b>-Vn-SJ4}5{5P(lUx(&0_rKNsr`G5TS`(&=n~586Bj(^%+=e@lUvEjf zRII1mC44u^^%Z4X%;lulC5->^J{oXg{MiKc^bzbd^6du0jIDF*19%9J;4wUbE&&If`p9@2(#3?U^lOr)2V^RKiVHi(N#3WpW!ONZt*O1qt z{lt(kot%jqF$d{mo?pMUL+HIL`G(8YA?+hr@!MhN#$zs)|C9r9^6R_oRKHXI zjO)8RBCdLoHJ!;PP`kkUCZ9&lJadQK{|)qKumZC8GOp9R*HS!H})skV2rcfR%kwkm@xJ_U2hG_Z%BU;kzws zUraZ~)04_>B9jcN}V+(}OJDz?*m*@8Df@&FB9kj;Za% zh4~-q*s-i_@r*7!pUs|+XPH8p-nrg0^ZR?!c^@C(L&Um4YPNQKdhc@fe}Z*>=$|3C zg#AZmvHM?XjDWam>3vRqiQ3i11du~^xc^f3<=Dn)=5Yv*L{eJKXqx3di{1aBulxUf zuZ?J$C0~}PBa_Y3)Xxv_(Z4RgMmv8j;{2fOPmK+rUl?z{QT=?qGGXUAA3a;9ey)Ds zvPk`ZiTXd;&JXA~t^SWRy|Z7v{7QcDCB}tH>ndD>>o6UI=JU-Y`{jol$N_u#Wxp^+ zfIbJeB97(h;a6l;q;C_>`L8T8=#zf?2xDx---WwzFVaU`1Ma8SdB^q0uW!}uxzO|N zKjy_YkJG2-n1}EP9>WuO3Qyx1bWPy5PqY5OlyiQ3-{Et9d-ec7ejmSmMPX=N#g9k( zJbwEUetU`a2WIi(rxu2Ob0YiiVecO*_zaZSU9gT^z^8_-;;K7<6C_G<&JTTCI`80JyoWe9Acgnoz52m>#*7Xh(8nXUnE%fw&inuT z*AK;2^MgMkKS3@1Gx8*A=nae9*AjlzYeU25;*#UsSF!u#KgV_c>RR1D8V)#apLo}r zAd`sxe}4Yg3H`zN()n@B?S>BPZPJJQ%Ka;6^Xos7z4CmY@;GybeXo-LE9L*q?td=( zi*|G%g*Z>R6YZ{JxO7HhG{zz=F6Fw%)8n2p(f^-FpM;!n78z;x;VN;}p2Icdb*QCJ zCugE&%-8<^UTs#wHzG-obN`wSvJc3Qe=Ff5%kPcya8s51k0iZW9-iZzTX7riz+Jc- z_u_tZCHVi#)IV0Je~^8p>K|lQI7hbZm;cdLFE7@}{~392_c{N*vr_qzQEq=-zC0kk zhmidP|CxLOab8|0gK8}Hy<#JxDm+;iNEqx`Z$ zdu(Vk6`#G-y7uJ2?}mmC$$st7ACVQmQ%||i+V=F%a1x*6OPs}!SJZzo5`#^~|9?8n zejcO4XyG{jZ!9?;6EO*K&D5*NYj7Q=qjs_Uzfjppk82t<;0F4QXr3bf&zArB1WhIK zf7RFaJ;6?7lz;4af}SK}`}cdUDfZ38wzai;FA8&{aVz57&)di@cDx(;b#~ePJ~ltI zl8t8XbK3k{l&7s|Q?|CVtrzUyYVH4sebjeI=Ptx?Zxz?G^FQ&elnf7di_4F5Tg8{2 z$L`?~*K@DacnFUmwt?5+F?zIp4S0h76rxZ6&Gm!p7P_6h1$nDbEfnC^xbUcP{UQ6$F?LBCqCy|>VZQ%_1ux?!Je^mcj>K!4D9cXi0 z+<&;H#(B%#7s|C^_QW*kaW3Hb`^@{ln|&jd?0>g!LVX}V{)cQnpx(b$Kl8)#Oic5^ zq2UekO}veF(6vhbFO|p0UirN5u)KvVJ-3(r&+z}H-?pCrFCTU+k~f#g|753d9RKsK zbjmMQcPHP+`SCv=(9f@*^&$Nuq@joZY&ygSh;~Q@* z8WO$~K8xfb<=PSD+_-q3<}2=$-!Kw`O~b=zaxBJUBC;P}9ww1j;W|u5|JsYfOmg5s z&t07}KKFHKFICQpPpweCFEciuUU6ly{BG|4Y4(8)$@dQ(R))8-4;XwkggK786}RCI z+=aUl_d6+n{PNKMTm4_casJ=H?e_Ad_g}BhPgYCwA#%NOS&xv9q4t<}-*0RJz0ox_ ztSSgk(Vs?h3EvWlMQi}-j_d!|U*0fV|1+AV>X$}xr~c=y*8O+RGgyOlsEG2kd)9v% z_bTnG*Z+^6`TFnCw^;xEJpK0z^xu;$C(QppZT|mWbN$iLt^XhCg#P~;;{(R%-$xtT zwNZ8;rOh(EMF0N6D?-;w;{#S3A23(?=&NL3cCs&Ij*q*^^`yPmE#x+A#|~6{q21wk z`Zu+C)*DyQaM1qr8T->G>|G$uB%0Cgo_9HZ4{FhXwEIrYi_gIKmvV1ti}A>9*8g9o z|KBlvuD?fIb*VV=4b&d8p91+dY6?78_uq(jgx^JSpXa*YbME(?9b4xe)R!6ej7Bse zvC%l?R(qY`J?FfS5AY#A!YB9)C(*^mcC)oTWSk4rM`nbxWX|{9vfJF3gt-PazRzy{ zzq)10^V1)ap9}N3G}aY+pNQkHzZ5=;A-jD0m)b{;EU(j-MvldJOvEJoT;6M+KQvrL z&+k9WR$oI;i%a1;dhb+a1DiXYJ`=eK^6xbH7k%0gZV*?!*?1jt4&vS!x01IZ?xE43 z4&QiOo1^euNFMt-|DIA$p=Zm7hP%bzi~I2a9>ODd4Bu=&ctZFoJcIHd7$;A*PgA!e zHyC$NaVH<0o)(wFCVFr5?N%3rE%a^3W#nCCy5;4C{P&+-5w?pT_}vv@2e}LV#$E1- zagQ2rh&AYBX>OM-fpOc@9FTZqHsHMlTi(d*imMI66H;I+L7wpeTAM$HuCPt!*t;)~;-~2cBPdK-p{gdXm#{aW{ zDWs3He@Fd(!2G;J=I0&M7I5770KR$ZjP`?L>Idl39^XAh{bHK-12VsEKbaNIk)vJ1 z*mymzYdkp-lW-NTLESj_$7VK2yHUDLWI~!r|JmH_KMRdM-P8i2nE8WH}!pKmLB2=Q_(d`VsFJpL9L34w4^d zNM2Zf!7*{1e$y(?zs9>sct_;gWAY}}osplf)*g8MckFLvtb?>3Lf1a||A72Y_8yV{ z=^4l7$9L>h|30GqZwmjz_uM|2|A7?J=tO>8oBH4*(s~TJeSBJ7-Rqp?+<5?9S9-b8Le?ObDu$?d4AieoJw9wW_FemLjSuUq#)nP2$A3LMi*onpizkH_s{S=>9y2LyTRAcO z;^@TC;J3k#uMXQ!3=hBb?p_jC>AmlmIwbtc{UrU~hBW%G{a$$Gh_OXiekc5TvNasl zYj@5PH|=}jSN6r)ar%l-HT#P2a_bf0mjz?ZVf)umsn6`Ch2z67^qXy)@!hbs^}FH4 zssA(leBu8L1I7=O4>L~xa^q2oysuf_SBZDxzp{8E$}8=QN4|r>JB@p}PrJv1_Qex^ z5AWjxl>g*A;X`szd)Hw3*W>?M*!@2l|M##qA-{cs&u|i-<4a@%U*Q&gzw{xm$`2Tc z(HM*In22iipFP;E?0r=py4Ue@_|fecgQboqVcFIb1^#r5uER=C?S_oAcVTJ1Db!~OIJ@DLus zV|W5j;c4ud7uV3TPXvAU0`sZ0`}}9Y55sHbBi5AaS1Y(FG|nptb@OM3`lGXzxidm@ z>5R~{#9X$5>(#%sKe*;N$Dt8T6I>UPMXqz2dGgX-hYD=M7Hq?I>_Ats_7~5&$FaR+ zUx{ZU&&NKV+yU(mi@cvD?vu>FpIM%*h5p_kr6 zuXvu{+NVz6FW!29nbY=vbnd$OY|p+T>&9v)q}N(|v@NEAnnS)DG>YpH&LWvAQmz$+ zCg&tF_J36Wub;>NSfZZ3NV&0Ey&U=WhmHIV=e&ux@ebnNHSdz|;eB*DrhAz>#tQxi z`FHyd^FdnChIVuyg)}##&bc2C;2}JM$M6K6Le~QJc@g`(kbPdxKJ)1_ z%h=}?>@(T2pZ!PM5x(vr{_g?ye--;)%D$5^?rG^fgEd%(^l8T^$1CW)uCE9A_W$q8 z|FQjl1p8aS|0Uy|o7G$SVRMyPsLdF2?Yu3>um3ZjUBx!x?MNvs_m8*f?;y8AT|&Jhv&#LiHZMS2 zZ2y0cd>^&qKOjFuzWslp|HVhbpCCEe|4wxsC9cE&{9XJ1LiK7S(TvZWa}wE4t@T5m z#gIL|e~iRGZU3Jk|DkP`{HMPDx9$IH<=Zm8NnHOTjsYAk&D0_JlkYN?9>@QTC(qB@ z?c)o?zQ-(b*xrjDB*n+_y`EJSn-)kcx2S!=@oZN*y zs70Lr*FYxGhIC|}I^I#=EIq&8%T(Vh{S91Le_(=dc}i^mHNSk;9M?8SJ#|iK5>Bk2 z6Ow4|Hs9DeZ{lsdgKy6Ndsn#2G2Q5ye>4C8X8!-pA%pCKn?r8y&7mb>etOFM^j7oJ zH=CbcWqvx+8_iEgwcq0Wm-nRgK0ZKg;T&yObHa!8#+9yZ`J8k4pFeeB{ei`X9u5YL7t9O0=vtzgG zLmHjvI_mz8yFaq`jQR8Q%#k_ltNHVu;gD*10V6RQWAV8>GM=1>x+R|Pai?+qUoZ|;++DaE_u_s$fWMpnr_P^1ew_|>;vvU8g2(U#V*URq@@YJS zu449=9sO_T|83O%hqkHmKRVc^6w>rg?Ee4^HIiHW)#AUx27AmyWZ@AN%qoeo% zzg1U{=bHbQCEviCcpLAa{QK%0A@asJ;2 zp@?G#rVG!+`T2i0(9h5RyOBNz|9|KIrCswi&db-ojpezOejDzlwxfl220n{!(H~;S;`uX{PkI)~(-_HMgLR`DHgQv)+kv?%zc!peq6u+*=Z*hN{ zb;1?M^7(V<+ozsFb}iR_fu4=pFVM%9W)R2N<>xPsQ9meFKVXB~9oNC;rjVw0E@eOG zv+s+(QGbf{3N|>olAXnE)xPa-*pE%FVGFj!Yk89IP{~K7=g0ruZoMgbZA$;edgCAH zjjpA^F}3stG@p?+5@;%;i;MIB+Q@XI#J!MNd~^Qa8^UkK_)_(jWy*dsKmRZD|2_Zj zZRfm$ckv!#{(qnR03YJQ{69Ybx90z~8lQ*%mHB@kN#_%MhLfmWeQy5W=k)XQZ@#3T z#ox~VD_bFdV@QqXT&UgSC!@kh`e+Qswe{5x`Y$)fPJQ)vx{k9~mF;q&9$;&5iNf-MAO`BaYvCfP4s#;4z#Z|Mvv_{P@48=+EHq z$Nv?D;o_gh8mvQp{day^1^xW`@A>vW{Xfm)_@+qk>zc6E8vA9|*;}{U`u{bf!zSmX zcJnXIwckSDhVAH?XMBM1$bDqSIssWSXIyn1zqFoz+OU-WTgv}kChgRBLNfE6&`j=d z?k?;>EgFzS8`9`9&bfP)@dw5o^pbtX9c0ExA33xf_>T5PeY!_nAK4+Ca$I_)_5V?Q z%(>FZ;tjlsT0U~kaciy7&#yo54*gv;Ynw>&)6cIz@SeDD=I{4~Kfs5G^Y7!ln~&%l z{-o`Wo%w|R8BSum@40S;``@SzS7jX6R&BGsnS^gAd0P7)%3lrPbH{#(vlvp#USK3f zQb#}b6mS}_rm^f z^v*^44<0m^!1@evZhx$Qq@@|h{gkP*#J-Cj;n;?sCEtj;y~1MsWAJyz%?Rh$A7KCA zp|>k*-X&9|#!s(Or>4J;574tv`HVjPLVkP-nLEzM@!OkdXf^&D`EhZ@>|KP1` zYzF9#<_MK?khk4-o14e#QFE?2I^B~DRnE%lxFR2>;H?tf!?S-5!e4Of7Muc z;pW(PpiYsHev|%*IpX5JLizRoFZ2FV_CpeP8&VnfK3}`RBK2?KyO5pjxub8M_dwRU zu6lKb{5m}g+4PlcJCfbu^i-MuuWzPL-+l(!QvLs{^#AMcZatv?oNPa$ z|DQ|=r^!yowV#?49+S=!cnVJ=UF;Y3SZT;wL=RZn^{rHp zbt%thy>vE9hivj}5}r%)jC5Rki}=<`_lNcx_m^?)-*~_I?_;6w192^;zb${;r?{Vs z^+EZsJ@J2$o-B=7{*&Ij()};@fAluQF@+cQKUnHHimN7j$Si7?O$u+2Z=%Mz`E?Pb z^|tUk$nSq}P(8!{#{Cca@0l3h6+iHkk>Ne^eSClq@ew{j)*d*Yktgx<8P|m`$-&?J z-{CCzi>1@Uw&H&aTMu6sUYvhj_~l{!Wvj-9$|C(`d#8tA73)u%J3ds+A0J*W`fm8O zYuYvDdts-3yH{4LJM6YMM1z&pBUxwC#FN_`eJT&)z9+Cxr4T&iyL(}n-p|RjEp+TLl{*bYqM@H+5vA@;+ zk?M0J^~qcus{8p>C+wYe)YzYT-&Dc>2>*`Z^pU8T@o!2M-s{e7x_^_=+|Kr^6h4QxwLwUF;Y+tC)5HBtKUZ|WsG3+Rv zWFDM$O=*usnX+Q-qJr?sV(p5)hwNK?67dr;30L77T!+EmPYTn?{;TZ~K@L1>d=Ysg z=HOQJE>}NWEPvDQK(1Jsh0FxDex)+!xHX-{-;H~5KhmxzCH?_=b&>slyvK*=kKqy2 zvhOwGpP)Bx)Q8n=Ttw{ue^&kfj5!*JYs55dEeva0?H9<8UZ?Fnj_-TQIjP0k_h(pp zhyD!Kpob0ZMISQABFC22v6uDT?B6Q(e?6PJlWk?kliKK;CrH=171)F=*oN)cfnE64 z{DMO17fauNXOTmTK8aSet=89o4*ECeCsg~dJ<_Q~1CmiM<$sS;XP`GGTpQB#9>hHX zlEQI}WD~OD-oTr98yD;Y+xL#}yLcb@{fG4{eL(*ZAK??!Z8rX=O8x(^`hU!qC)EED z*O-XwPsjTIXO26G&+#SBVn_pjz^s^f&`hz516 z#zo%aeDf%gJmS6Wi~TF!;~{e!UE^qs#du7_BwU4S(0-@cHf7$8yPB;-#6L2j{eQ{NB-gd!>#K7 zXhvLr?=ETGjeBuF9>7C*1YI-OfAk=>wg27xzk}-kXhXc$jxnA;(&$9LzWv9fGZ5P< zZuw4lf*$)HpNes>>l-7V!5R$e2l!O`%etsPYzzUp30trY?cL^Hk*Rw1e|dZdeHZp1 zmJz+^bN?CS$ER1R|E)4_!tb?cKr-sBMWJSyeq4HE<+;B9>}%?3!g24kBogcS|M=GY zzh!)B$E3(KKRip%&;Pqr9gIKS%ir%)HfQ+ZS^jvAtXs_g)ppQ;Ml>OTB%1MtG~UG9 zcn9y|J-m+(&{e?yM-O_7rC%idS<)|w>HGd&ODliBjj!Lnfd9XU|G&ib(L3kz|EHQm zAf1o!2|h#ZWd6VSlk`UZeuMUm&*@(xjva_=118A0hum3lLmK5pjKtv8MPW2K7UMC% zPn$^gUuJK2a_uxw=T zH{eG6!})(@@;`EDSuFpf?YQz;`S5r1|JdI-(zzA4;SQvIN2ykJgWkK7eZk%Id-1pP z|JHwT%Gfh&(E05F#JMLAk&mF}l>B#?Uya9vpTOVF|LZ!Y{6f!isRqDN{tgFdf(tEYD_N=n58~q*RV%q>R>HvM{fATxwUGW25=BT`G{=i$td0+%$fuC~b3u5TtoW00N&Nn2=xeYp#$gi%TZV-#WbD7)Mh^UBSlCYP zz%J}TUDen2zsUP2l>dvpFEYRX#4Pszd1Hzk*MKD2kVX%(=-Mm)pa;F^Lk3yokYE3Q ziu^zM>+yf|IJWh?{dq$gZ=!uOKbws6e%~SAMOxf@D<|@BkgY;J8 zlF8{vIi`*+tVhFQ*M|JMWUE~BO7?$+dwTRt$iDVpVW#76z>S!LTX7q@PFw!~`F5W9 z^7ImUd#*f=Tq*y59Q$9){*&$d+5f}JR;1~j7`T%EuMQCRHygNQ__xLn-XZ)9TWFj zN}{gxY^X0g>$kI^5l#H81d`ony}PsFUg&+rlcvc-%y5g!ANssJ7?E=4(Z=hzL^X9uRlwUhEyeTd@+kK(Qaf!KS&2Lryv%%}o z?z-M~Ov-hp51kF~&?_EfchMt%_R5!i@@Iz33dj8&-*wD;cpo3&L-fCXCVWH=yzo`{ zg!~LAG1&1{_?-L_XE7ux|4;S&(NN+#XSpu-l348dyS`@EKin}RF&bkr9uqMMSK(Xp zkJ+6I?SB(o-#FjjKK9=|wvp{e)c?u+{-R{(>Vj~MG_J#RRCk{VGs&8RXF~0vGok!< zL&IF*tT8yT4Y0A$x8qw*kV)ZYvTVM5K35ye2>$F6DP@L{4?8J z#6}}gz(yDIBgv$5TKbI-LL1^-z7BqQ${3&g`rsw@_9-;Jzg97ULsj#u9QVmZ9#1`oU@cUGEyu)a@FOq&L_2&$&awa>uQ}O02^9@dvBv=g04r z(#w#WuS{@GtV8$7KkLP9#Aa;8;0r@TC0T`?7+hj9e`Nc%?P8h}UX zy(PX~9HSpcuAgo2U+339%#Z(H$j3$F0Y2>^zU>h{F4?@F?wk`ih0{2Lg66Qc5F_xd z_5aif^8R-L8;$(>e|-B^{CDgBRUaN6ilj3J<1hhn4Zs?dzv7$Yi#H&)9~b+rnaxgi ztAnUJG?8n4fAJe!w^;p9XX`@r_^o%U>zl3afGj;XSskfr zc$n(AX_$dosMR-7qb*}Lz4563cT5R=E}D;fHsTVeJS+77urSOMKXCQ1FrQq2g;<2e zSc0WkhUHj+l^Fc%u&|1(JFWd=g#Q@h`A?JPY-vyNAM|E&wPQ+AhV?k#PZs@$jlx|= z`JczVW3u;*{7+_tvt(|u{O>uo>PKlqJ30{eWsZ9ccB1Ye`^O%|yxAyE#k`pylW3kL zpFKPzZ1q2tsKQR{Mh)uGHAViM=J}Gn@@pTNU!O~!%b{g~{5@Iz7T@mo4l?EVG}-CD z+SeO{^OiD2T5%3wE14FTssug1uh&N9$i@2WUy2Uq`^f!hOeh2Py0_iu%7A7vmH`L+b`Xbf7=v#O4M)g+9K}G#&~Pk<)xVFE zajaF@D&Jda2q%P3;WT33UUnGUP8PJtABb}RD{d(Y8>}bTd&)cVelj)QQMY#^-)89} z99M)fDA)cmj+}sE4F0wtOeXukKRiq!W8X;h|EJQYVFqTQT3voNS%NtCZ!S3xHS~s+ z>XoYt!hGQc$orM-LX)~>VygZpW5Mf<9d982ZvDT?(cz4HTj;z+Sd1kw4=|KrExPpM zck`w5{rijg-}Cw3$nUeq=WaP+{P}6)&-u#j-R1|>pBvks-f8}SDgR#^K>ZQ^{|Np+ znkMl73;F;2_-1mc|67LT$Qqluf?SCWKl4rK2kjc?|BLy|42_9@$aGbu8CK>N9K^5g$5_3iEFAE58B=YuRg=eYBB zxqgJYaJTe}-@?SRQLB#oean5gC z`#-J$8q0#t+48>l)LdnO@EPgUZZ%%LYG^2E^(~WyVsZ&KsQXk5 zv!+uV!#8Gtx&vw^}5#iVF_mv`RKOP?uc9@&_m8J-zj}^|EIpX|113B#HsM&nbTqG@zY^l@qg+6`ZARGueqog_j%Yb`}0sf z;>)o9#9zavy?+V+p>J)T-{xaW|ETcuaihZvvrdN11=3554!FhRPEo!;XW)!>^Pzi~P12WyM2f)}i{pv=f|?p9?OLpT~##&ErEInk%R4&-=bU zps}H;YHX;knH2V5x3-8Q%8LJ-FwxkPiJ``Raj)&XA}o>SQY^!AtiVdF!fLc1y)Lw& zHSuqu<>>TKDy|Hvf~&Q~Ul;0>U-h$;DJ8CjY?`L5NhxzO>ObUq$8N-CY{iB0qdSC3 z;jRVz|0VqY724NF@ChcI>wC-6a|Omf9#HljQU)GT7Lpypac#cXKHo`JNu&I&;;@^H z>-g7@^%%U}_jaGPKjZiSL$TJYJ|gCVQ2;J>)*W?Z-6i!~q<{AsogL^rJ3eeSnm8+|f8i|96T1E!H2c@3?uD z{_i6F-_!Je`>kDHct@qa>z(?g$xh*}l<@%>{nKQx_`db}?iX1f!2JFk**)=yt+7?Sa}tN$-vP(%n-q$iHwO9f&yN>` zar6l&#$<7yHmD+%U~^omCvBNNXyR z!p)VJ`ws2HFr`@gjy@WDG)(i`4AhP(4ztMFs4*#}ebJ;)LZ6Fyn2+=X&kqaey=2eq zNns&<5pw;mjUCFkw!VT%VX?R+Sc+=457h;s4K4 zSD40sU3Tud$1#xQfluSu)!6W)`q@{*LK%HMHexfjq7qfui8ywBH(7&vB+!Z!2LEX7 z2QrS~%a8+?7lgg!KJ3Q<^se$fFIVS~Uk(cAlr33g&?jHSJaZo(p9zNGk@Ei5-W5OqJ3SIT`KQ6TYEq1MJ zYZm$TzXRXs|8rc2I#vp4dgpF?4qa=k!w-ym`l6&G4|y zam!I7zF`@^dWCY6Z?)kT`CqtMI6u#xT;aIEXA8qhaursi6lGYCjo6H>s6-V89xV(z z$$srP?FWq|q1T`u38dX;3a#|sde3IJFufZ&_C6~vgFervuKVlp4-;G`^6T=Ic>c3o zr+dse=bPh;_X_XBejG%Xx@0%<>&wX37wVVFrQ9Or&lJ}&&GRSYno1q?RI%qj&hz(8 zSFH71{yZccl2)~GABV{r<3MY1gx-&%XgF?s=uu-n;@A%|iRMGru~UaX=J)d7$W!x% zhU4_umvDkSg*etOhd9}MyZdCs-Q}3r&`&LJgMHqwT&E6RjXhPnP zPYeyEzRPibpMYYl? za%Yqcr<4(e-W4{B+loq*-!dvxkvp*)gYtR}*?+Zfgd8xJtKvoP>#I?rRXByUe7`vE zZ5{uw`iO5qT#fW={ZEFz7yA&`{%AzgM)iLrU0<{7Jm4La_3_n=7r{4T@^kw+I zHGE+8#vEB!#{WkH8qtKrX8!;BbL0PyxQ2cl#W5Vm37o=dbnWE-C;0z0{C~dvxAq^@ z4$*Ru|Bp7bqa(%t&+z}rPUp4rug^%QAgzqW2;|ov6kkN|RoCso82UKmR=f6yEk)dUCG!hNgwy zBT8K3T+G9KEWko6!eX>9EDB4=rC5gLNUya1!0Mu~g5Jwk_UJQQNneGW`gL~9xjJ?q z`?*?N^|+!?N|vE^R#8|_ZbVIqu`BFj;~C?4J{}sjii>?-&Fo_n`R6LUQ^Kcl2Kn)CS8C(w zl+Q2%MW|zg>r>h%GGC7`7@=){2LB(;(~P^X8ydzqZX70{7?UvtQ}Iv7ziAt2ox=Z@ zR(lEmpG=M8{}=QBwG~d2M!#{_GsuCz7KT~mY?NRw=3zb-U?CP^F_vKPW6zwgyKb7i zubvq1eX01kc0g*cdI-Ij?e9@XSWeIKAL5+I3-+JwuTBjQEB#h+t$coGL0CnvouU7s zK-)jPajEA&W_T#2m!Wx~=fB+ZM_j96y}0&_b|i8$(#rL%WF?{v>XFZTmlT95;hl(m zKe?mIiFv+*Gse%X^!+10|AqY9{cX!Xy(#Q=UJdGzKr2${M%NPlrh4{&yZ*)@_1`1< zzsdF${NL5;364+GJIP|-Mn*b&5$m%1$eJ0(vtU2{01l#|SpFR6`ikVwlsZ8x|6DyF z?|+P7|0aZR$Z?0!kL)Ekg`;G}>oCGx_qw~fg+5-I7~n> zzFq#xUxz){eexHgt^d3Ee}7Z{_P?fWNIC;Q_Fb&KDU@G+6M1u(B5o?CVFqSlHcBuT zaqRy*vaHm1w=#tJ^w=k~kX#ghzcYlzWNfQkLdN+|OUY$ej*9zkR*$$TwC^klM?8lG z;#OcK(ueHdo+=5e=)I@y-+rPbtfrSDH(`$QcTUJSrVnM}st=ch_2fp>ZY>F$$*rhy zUW0TRGbN!?xC+TdbCkbxLX+bXs1vR?m%m}@oY06SB#=ZiV*cOhyxlQ9)FXjbq|i0* zX6-NL`J2n%yTZJF^ZhgC`)A3VaLeACLo3?Qjt-=dMkm&GE8h{HUAOf5*Ukxh$$@K2 z!alM>8}Ei;#-LwZ67~xpz(E|sVYJu#Kk^9raTMv@{zLpRdhfhBp$Et5Cy=9Ok#XOB zs9t_^I3@lx&Y+;%GhBLes1aXCZ(Q%$USxVc?|#lwDma- z=-c1NU)Bd3=i|+g&W0xS=e}W~i#_h1#zs%k{&h_I*J(BgacocyEy|SE)$;#b`5ztf zb*hAYn#KQ}%1>QU5@tznHfl>tLJ7IT_gHPsq`CCOW_cdX{cQ4ay86&O@%eo`68z^c z^lb?b{FRM=eOOpPUx-Cmj3ro#Wmt|CScy2VXcbwfOsRJd4X*#f{wLG?mv9^-v)VDG zD8qVe#AcMY3=3Pyu7mFXu={s?z3AKO{&%|nO839f{Y$IWy|$sf(ETgNQYYL$I??~L zo5R2l;+%&$_7At7gR(5H|6M8VxQ192xf?aeUO6n(lY@Vl69#|l8=$wM-E+>*e{}!( zy$8f~BPG7aGwMybCgHuvy0+X(*I6UZHBJ+^6Z^0q2XGLFa2WaZ^VqNkAr!%=#Tc?PvOMn8@dIE89)?dl1R;TqpU`n%}mW39!Ai6XZvsVH2ZJ} zMZ)>@bJ^3(1pfDA_IIN>2FmO*tkrfm#xeQ%f9o#_hjj$d9s&GU7g zjjl7+sS**V@8Cy}O@2|c}`}=;ry)q`hFZW*d z9nIRbD;-mXo!E^U)FXjboZEl=>+z4q`}V2dWRR^=&q2#(?eFu>^+Eo_{m19ZkJ9Nz zZWy13OvkuI>d*AJra{kq^%DC2c#LqiO5WKj50MAN9mF9V#^7tl50L#hih)biJ!1Gi z?W0!~`7k^m#5SE1F`nJ;f7Sf{SGDo~_@c1>rBUIO_`%!17f#3DuO1T4kONOn3yAcCA;f2RW+9cRVB%iLa5r$B^StD{cZ=j5r6P!L=tAdv6PsZNhPnr@CeQ zwpIL3{``gg?^66YetKO0|G592?EER1ifQ;EZo)0-(y!jVg8#jYAIUH58|N7o^CQWe z{`?m00Ii3#4;;}xKz0bHR`Mg!>Adz+qr;D+b35+DkCC3`82;ov^xiT2Z@&0_^q(Nd zH_ZxX^qcpQdH=ih8~!)n`9a6!{qNoGix1woXmof~T)FWRN#W)b{8KV9Z*=(eyo~t@XIMP!gf@am4ugyO2RKT-W;}#`(fBxGCRD8 zpD&vd2Aiyv5Qc>fuNH*I{m+y5DF#0-2tOnHjd6XJjOF~=D2-()kMS25h26Ug!t?ZM zW#1lk;+g})!fRu!Ay_;i)S`ZOu{MGUZ1d&DuxWeWIwmyfGixrm)cYF~T2GJD#yUT9Q}VYLzCZHu%-0};pa1M2rrb(44bFT4BMvM5Pos+;!yr0We5AS zecZosK4ScjtDY53l2uvJid~ z__JfG5045Tlb@ni|L9-HzoN!@4bo}Mj0#@}e}&{C9MK8)yBKT1B1=x6tgVE6H)@K2FtPjl!)2EX@AekA^9cou6>j_2_L>h#^` z+XqtG2adD3&PgCiZ$7Gjzn*`Ow)y;fbS&oIBTer_*J|?*=DCi!u49S+rDqlxA3)}2 z8~;~4JiO?CUc$?G1+|O$-|Xoh|yGO@41d`OEAt zzpH~@5%kyo#y)}cKD>_K;4So3yzE3(%5 z|AI^}^8L&j627AMu9i1f4w0zu6gl5k7U$QmyjWaZe}5Et32Mg-373+Wp~iU)(uwuz z%Z0B*@-*9nCdVakt+?+aJ4~N;m=vz3{{TP4O<2p8m$|of?x&m|U(w=uzBDx4;S2Ynjd}FI$5{-&|aOOSL0x_(O51uvY)eem}qd)+_Yh1D+pVrN4$8 zJ&O$b(B}QOFH?_>^|VFm&86ziJM9xh#xW#4n~kwj56OjMqJ-)n7F#h`seX?`x6xCpGON?(S~+(6zcmPVf=x<{mxm& z9~2pXFwS)pyAHBXICIqagX6{@kkwnS2p9XGifh&HZ!ZX==(QV-!zjBVTtaVjOam^Z zUxsGqCHw6maLQf+xLjQOv`OJg@@m9!57(04M;r&y;~sl)z3>l^U8P=Ls-C{vb)fE` z`#bFZ`i(z8(?a79kVG?n=$xBy3x0&#aVLI^d(d@6{%7CvN#`ec5D%mFp!a{cI6O*kJmvl4ar%?U@2BaTPaO6Bkzap) zvGPX!XPy3!pE>4PtVMa`)1l#ca>MiT?x(}T3-lN95?)5#diDRxb7T4%YuLq%zP>~1 z{|DJgyyBQw@mf3E5dI_ngpYB4 z{jE>w=hxpVzgph?6Z?Be{>SOTKgFN% z6+%uKkBc!1m!NJX`-BEGqDdJQ*UinZqf6%3_dm-2Kdu}jJI*M#$n)#_cOBvXuVjB$ z{bT>X{qXQ`ssFhQm*YyLoge$pucr4FC<7gHE&cn*31=tp-^cLZ$?L^ctBd@A{2^jH z!A;~Xs5znTlG2WlDGWanz8%Sh`mGVyE=VBmof~a;et)C>f^er}YM1GEExjoGnBG{% z|94DW|0mnTCJV=ev{ruk>H?bNQzQ;Zf&3jwkU`{0z@xExOq7ZuE@sKZVjQ zmj4Unf8=KI|K;-w{eO=8mi=EUeQ7+87f`J}_aa$yOnVF4{1W|Tyn=6yk7?yQBF>M0 z)o-sMtL!c}#-fhigcfulKVDwnNc4BRg!}M1Ht-i>|9e~$_$}ez;&=Ex(k1dJ{y^_l z-|iVRH2jhNC*->2?N)i5>?;@={wywz5BiuK&_44i`4{{ZU*Ic**L~Nx7^83rF2&%T z{D(!_|K#BYb>v3;ZU0BsI$@VN_HtZ_tFcagig{`+x;(>feSjdHw(G z|M2{M4=LY6zW(LB{zuh6{=)x{>;L@KcP{qfu6^|MhU>(7G=(VNVRdF9Z&S`8|I3)ap{vhI-a>*0wm}Jvj_rIS1znPy*=J)@j zC;0x!6hE2I{sZScj7RY}p2Sb_Gdzo~g#LZU_Bf}P?9*qQDdnS+IsSc1J^x>Po8Q~X z4#&r}{qy7h$+-XDTIrPIdAxvhiDL>!hZpI+{P~_0{7L%D$Su_WzMTKR)PIpN@5cTA z^8G;c+SQ}OD~@{=HFMq9RQEMY|GTy8UlW&{=Dx=1e+CVBZ041{)rhPo3L@7IdHseNnFV{?T(l{@dsMAMyUleE*NU*CM^v&F+7_d!6t7 z|39SNdvKQ7o!I%KTN^x5H;lmskLZR8OkhT2U;?wEBX!disG_&%TBtx3WMLO-p#r{0?h#5#A3khsc1X-9s1PLUNf%bmR zi(z`~BvY0B<6FPqInVQ+=Q+=NuD^3z>iNmqRem8Inw^7Z%YV(`nE}B+Hh{3;1-c_IZUo{ZK#C zeh-sfE5bC)Kru=%8*?xZ^RWPnumsDn0;{kF>ri8EP%Y}%2@2V*!F{jS4?a+4=o4k^gZH!#u)#AnVGCMI*=fl3sq72v|7G;F zasEAIcNhBvQVr_vo$7)Y?C4M2^P%`J7Jrln>-T+w^cde@llcM9bHiXsPN?wvxc+ZH zc@T$j6vuHAHAtWtZAc=8p)1n5JU*~V8sU!X|IU)<(T_{0e3Tuokk@bnw@`gg{vXH= zx9Rnp<$v6z-^c6r8~a~Ec~>(<{e!yFU)Z17)8p8F<5v9yl>1GzvAwbnJmfBy9v+iV zk-VUON35fJkfQe@tr!U%qqkF+7kF{s}6Mkpl5BkrMFw^zI{k$W;4=}kzW0U7)I z%3svK+xf5Nzh3;wrekk}BEOl287M{xW@8TKA^Al7wN<*wU)eu+$A4x1qk6y7`_C7~ ztNDTp=y8tZB610qp>LM>P8VM?j#s=I&lu&Kt#G}v!Tf`*FVkRIOoPkJ{o)trKCW>+ zp8Xf+cJ7+@N!Wec`T_1Zw=*+;VB;HMoqL9Eu+Na2um#&thH~tQ^0+ii_D$Rx_LGBa zz7h_S1DpJRm$SoRdZzv_+a8Y6JM{147}s%n9DC|kN1deCpif?XZ9g{#RhuUb%2#o0 zKaTOoxy5l`>?U@aOte0bO?U$7iW7!VLj_bxXr>H*;#We}Op>$r|t34vzepIz9cOEErgnfzo8X9jZ zcf4z2;AKB$*NW|7_m=I(HnpX~Cn28gdc%FU5X<=6Lz%S0d=$oQEpQXLx z+R%+R!b9>gp5i&eKjc46{ww-83>fRr<&Jx!4U2M92>IlI_GaaUtT2K;3OhG!4P(ga zGv=Q0FQnI#b?36ec=|-dJwO^;l>hEYsIzKrZucL29O|+@4)sW&0gY(F74I_H?}{)D zGw_$z|4sckbPaqQy3vCadXYvSnm_nBw0!h&IJNZS(7O2J&^Ghq&_3y7m%P4W!&XxJmb4w^qqa1 zhO54hHe?Ii)+uxKTG2*tzi9p5Bm3JRsZZU7Zv4vme$wh5VOC&2R{5_El812=$5ErL zQHwftU;Q06$`|I4k&S4&8p28Ue0hFx4R->~NH1bHE>oA?&kjlM6i(wT&LeFN=HRA} zLqB~WxjkGWuizSP;1+Jwo7BX`+ncenMHgWEn1Gq?xJvO+Ogg4r0l{k1TMY~7d@=8^NU0E-a!R%u_U zA4uZZDiOr!6S z`fsNCzf}ECwic-WN2}Y%ssG8&;nu#LSQlFLr8D+7`wMoQu$P?YB+%Z%Zg7X)fUUoo zyA5f2+yf}}gdLy$lV@APkMC@;pXAnXbi%gqqet7q4_9tCez!3kzP=&+U|>V|{?<+5 z(E3f`AU&&<{;lS24VgI=QJ$yQ0efZX$KL|--r?BgzHDULm&7u6r+OQMfuG$>F z)3rIAcI_;_yKzJKY{#}xF>7PkhwpjT=e66zdDr^!{k2=e4}71416#tO?OW}AnP>f* z@g>);;2Lh=Xl)3$$Vz8u{nY<)0=Kz;)-pU)+o!V1xoann7}Gj1THkDRs5RbKvvGK6 z8aE;|-W#cp_HrDs)jGISj|xM}8MY$!qBeQzu5j81#<0bV4fp8}@fc6>dVc1Moba5x za&%q@=fr!D{-d_)FnaYQ<3BU=LO#8IEgQ-$@0LccYn#x9BvLqyvpA2M z>H5d|k9Ar4$9O$IUH|g+{QpV%xB9WIWZMLNcXXf=$#d)j`q6O>e2+eLitOc1+bf_C zt+j84e$TmtE4YS^%fh(+X1GC*X9jk^cr)Chr;GI2{5~DSa8JrTvENzw;LVWfhc489 zJM?C_?cTenI_AB*y!TH1c>Aiw{qq`kcyAWlUVz!-Y{dSk#Sq>o2asXZvsusEMB zmA!1Z?~A%|-gC6~n&7?Ah$c+*o5?7`G|WITN-!HqZMd$h>KOg`9x}zF~Tx_K7*dn1}hORNgEgtEvk0A)NDf!<;(y`z8G9?kS6qFlV5_ z8Gwywx@kVkT=mgM>LWIQO6hG`{QU>cy~PT8X8)g{9-gWmo+Le%%3ow7nnv4mV5vU9 zPS#zZvum#&thU9Af`}NugWY2bO06f43hYNmfjo+X^vwRc`pJjsN73hf)ADmK;{LkF`JF@!5{ToD z&15|Lw~frq3)ioRc7fQBPV!6PG~)b&xG&_uLG%BMVg7&Id*Hll{kViHxP}|Jh1HjX) zM?~D~HqoyB?^XYkP26Yw_Bp~olKwFa`51vwNUk-$j#uOSsj=#1qzly3#vGfks{geK zPLZwdYujpk9&vx3&XvaBT^l2uLX5{mRF~?%Vlq9hiHq~!i|EtP)XV;k1|+iHFu_y$ zTQB{s)_z{-x!3%^8-<=(YyJ3p=GcO@M&ih>#|3mB_$2^1l&&9u3 z7$umEIhcp}Sb#-HZj}GfjUK$-|MjYGEH7r_-y;5K-KYKIIqeJC|Impg!l)F-W#kG} zYfCNiy;sqz=yjvL*G7GL?sdqt%MNJ6#XkJZ zZFI*z`W^Z6h5Y$I{?s<_!(G?z;~^g7DV`&om)=mLOsK`{@`2ufMl_-M1Nr+S`FpOs zyjY&w>iev=$ZYAT#IvgGUGlYm0c5F?ps(z4)ff6R8PqcBgiqR zKfyl1_SaCW@888fafp2a<;D=w^TvhUtBn8Ow~m($tI++i92idy9nA?7$;l|fG|WIT zN-!I9Fc0&w0E@5$Lv6ABJ}wMwlRmiP89*z@Rak>{Xw@FyKyJboY(t#e+5RNUA0*$O zkT$QY4{jHRJ=}c_^eW{*i?pggP&3;2;aKB`Q;Z)Xv%Y_l@x!Uc56Mw}^Xap9jLE+v zQtyOv?h5R3KGAM#DU)Z6ucI41NTC;L^r3mV@k`;H5?OV5dG)H_jm;_a|s zcn5J9M^Rn%wsnthhvW1(CSG^s?QoJ_gQf@G?T#_dN8a@r__ZE!&H&klj^1}dl1!n! z-5Kky_24x3S){ib_eTo7sPRs<3q518_^%b-R$-%&-h}gh(~nEIf@`>eTeywnb@BI3 z-QJ}Kse9s&^fB>2B>rQ?AE)@WqRl<+=%9C^^5)Au3|_4tx+~1Mhe4J2-KW>PUWb$N z#6#}5cR?fK-Uac$ihE~8+fzC}D?D|*gZ`Wh7sOxr5^YXBldL10=l?Q)EKTp5>c5{Z z4ItNj!w~0Qr1kal=|eT%_Bw8nlZOXp*1e89TMwpf%m02OjvDV+d)c#5U!Yta zt&g0iZ=9u%j3u783@fk(@$A5LyBtSQ`mJ}+z$)=(;hZDbkBqr1ZYTVxNAhi`{H+=GSAJtqhB3HC>~F~C;e z>LC3vj-s3mpyQlp+{_BcxqB{qw~PAk^cwWFNCOxA_h-B-S$RD#tm2IDo#K9WM6@Bej7`|m^Z z0~ebgxX@fJb^q)Ai6?1)Xa}^aV@C*Q6vm(saSng`fj7c()mdv&gI!hH>0>e_qkbEtb}{vQ(b2E5w; zzf2o##|!P7Ga<|oM%?pk_a-(c`eXV0zqtPKfW8v`s{Q|mj5*I2)&eX-To<#1T!s}G zy!Dl^ij4NJHPLN6YF%_ovm3}b|7{bw1>ZLhZX5Y({#zOS)%>?|`X2o9{I?2z`*9GJ zFSN7Cst4Bp>8l>4AIC}5DGOp5kw7dXV%gAyFV8QiabE(>NH59{Npja|{p*R!1o~;5 z#d-AO65`lHt$bdGdL-oi2IW9z|K(Y+{w)Ys+;wPeBeVbVLiu0bZ=E9l3$uNa{EyDj`u64=J`~QNxrC3&r-*AXo|EAp%g^iH3b|x@ z#am$*nU4_|(iRv+mW`G^#!4Skjs5iNL#)uA<{#T1h4DUP1>?!g`3L3>Or{rM8fKtU z-?PoLCen*hg6aa}$CyozebRU~{T%u{G_BM(-L8*{MAn;OKEK$9UqCKG2Ym^-4DIx8 zYw&xpf_oLx^gg`WC$MI@{_lFvSgrpX!z3HH8_A}wx565~S%(eSge};HGL$1}jd_DYuAn1NJ*jTIM(Jd0o}FRsf07-*JpX0FT!B?sgLT+|E!c)K z4BfW&-`s$5dgU?w7Gpy@*ummnkmX0s-P&mUzdbwbaXqvDkGSoc=X>_&hS&3d#5;jZ z|F=p#rLSL4#yRZ`C)DxurkihsO~RR#N6WX3r>r>0i_A9dcZzJ>oq z@t-dLp$TOdv^DW5Gx;sQPjAsC#clds{FU=>7s`J)g}7JDFVDY?XA<5QPGg# zS6=Uf64T@m8`#b{u8+{BaR1=7-wC@Gl!o2L*p_h*owRNw%Gudr6}blMumPK}1?43r zVH;T%b?m{Q@yH4?j(@i<%nAGH2XPoj(Xln!fO5iddhGLePiGUS*Wi`itC#ID7446u z^8XI`|A_p5K%USZ$ejNv&o-gF##(YD{I(fwNFs&PQD3P1MK_|2Ew#~qgEac^E9?LH zwd2+Ke`W3RZm;^AJTJUm&DQ1LDhd7cp+#(TMb-r`lBT%__k?hTyoMXNh02A}BH4HC zx58cWKB~9M{~MLV^!i!iHa$B$rawjI{6A@=VYIkn|I~@$K;Go=y@JW%^Xrqt!JSjX zp_xVD`{#%@BL8w;(cwb_k9EDY~1_$KhCZ% zcwhhL{V>OG=3zb-U=fyJ8CD?azOK>sKS0kU`yZ6rkKl!G{^Y&T_u#$Iyxsl>AK3o@ zt!P90NA^E(U+2vCL*mESL+>H`KUh;)JI?+C z6YM`wYX5;r_8*vP{{gaTsc;MA4{hAoHaP6}M{yiAF|4G%zx>*zJ<>ha_#0B%D!oX1 zMtn}Quv%u8hEr(GD-CU))qX)I(tH?E|^4mIXkGQUX9rp$_j??~lqMb28J7cu? zcj^D*_4!|=%H~P_+o|ela+BYD*}k%cdmGA--e6tf4)cTdv48Aj|2V?_v6uZreUet6 z^eJbWmur70H&3AzZD_w|e$i#~i;%G&R%M6Get45#=8HYTu0T8kES~+UT>qGj}$MmoF+E-2=zH6uc`M%gcmp*;BUZm-L*l|pKh!vg{+t{nfHCTrY z*cHd7v==wgw_qE}P#%fj{)j$2y#o7j5FN|C0}j)B7JByfx582Sar8~sm;X?o-X3+) z22r_G-<*F95@<&CG4>C%(d!TBoA;W>Ku@9Rk^cK${r4UE>f~vDXK^0=`11JLCGK)| zm@DM&yKyhS+;EM41GjJ+HPT3}G*d^`Px9ZC{^k6C*FE>~5RdT`&tcJNSuWUULl?TQ z%LDTBuk3#z%ob(IDY8}AZS;6HOo!)nK2s+MBOfEsdcfLU^40#dW9Wq#k8U=(IEN(l zLcK+%5!bR+KgtdhU7w6N=e&rlypSEHku%WPtFGFoty!h+qSv)+XOqRQmEiSyPyhXF z?zHwq`IwwAk3Js@un0@A3@fk-yW$*sV{2>Z>#zZvum#&N)XX+WMq7C~8P^5xAuF&S z2XPp2-NRAxI8LGl0}1~F8SU$3)1}eUm+g%HY=CVdyXH@#;}BYo*Z0YE)01IS8xqCa0|C_7x(cH zk1=%ntx&z1ZH4|ERqL}u_$R)9gLysVFfxg@PG@$MS-yiv|jX=#2^)Kpb)l*1x zy|ha=(wi{KJzuVW#&8#6Jkp!AzZ%s4?dnK&gB~))-OJXVMjx6_=>OvsTG58MF0P|W z{qMPRymzz##QkI2=pChO?ZTdnXak6S{~qib#_p~Elcx6}wb64B`~F4lorW1GM)g+X z7p|4i>xCU{0JG_H5YNvgEm5y_uQ`k_d@*7iT_3MC);j{KiR?ENwz*PwlAD@*nmyw$aBxg zH^LTrX8q$V>(c3E_~rGF<@_rBPkYD;RDWPU1F~GYuUaf#+r?|H_fZZV=HIwnytor& zgX=X%^ncGt7pO-94f1cJeBFc{#_G#()Nen1W{$vq>+k*V$GJ};dCvIyMSX0t=caZ6 zz1J9j`iXYJv0seO8>2skRec$Pp`vxzC zaGSh~`xq*=em2U->P+%!)SsGvqAq*YKPeC6!gK!NpUN8}#)Vgy( z1j~@DHU5q6<;K|;8du+FTpj7n#@W{zmw#dW`=ashbH=|LjDL3-|3(MB(>Q-CeT8sV zVGR!YejVf7gLT|J!-YNT?XZC!_wVo9ZfqTy{r^WAJJ*J&eDU_nd1wpG|6A(1dSR3M ztLSy^t^erlm*d~c^G0DcAz{2fv;Y5cw+Lq&%Hs2}2Nl?l zgLu9EcB^(CQrd#O$h7mW%KtmXf1mi1t;fWlZ0GJEJGI4HAG{q73+E_~<0LxxwWEgK z^P%{E;JxV0=;Kav_mZiF!sAz2Yo7x$g=+d~@+_+8b$Q}dU>*Z^KN`84xD%trldPGb z4URh0BY_4qq6wG$<_fMMy}~#Ec?)-OAIZJ)Kf2}ho|q@QRN+q&eyQ-uH2OyC zf7~;V+4Ws()W6z_;filCU>_y^x%B$A;0+ljYch3PhXoe)1p=<0vxg z57a%!>2XY=8z<>C=zF0KMI0lKXT@jw-|Clm#$EjZWk$QQ*pAjRB;|zH#dRy~-|}(l`H=`G4-IRL`Y^a~g3T+F7!C z;mi4d=jpHZ-^tYf+oZ|fFU|j>_q*p3;@&@3$hh~x*Z{d7RrI=Z%C(C* z;UM>6G;%j_CzL-8WM+TwXkV0n%Y!)w?in~K?~sGWUTeq%nlUtu?SM=oh11w|%lv}J z+VS-B=*J}t^}i9WkOR%qBN@;3y+Pi>ZQR9uJj7!>#dCyzreB9)i2KLw9AkfQVdZlV z7HV%yQPzy}zWPfe_+|PZ{q~<*V;?}{`vaH$NlwQjeY|JJc3khdF8^Pa25y^!&K=vN zY4jqVRa5iG{%NQ~JrdLPZPAD(ob*nk{BArZVls*_4Kt9uXMR7r^@V$OnisF%+^cV# zMqj(W(qi+^mz#f1wz{wFnfd4Fpmz?KKfkpg6bq*WvoQzNE&AVGZ-#mF`Xl<^2h1~| zFF@QozYz^cc*Y`rt!xfU$YtobXRZOc3hnf6Hj5s%iW%auhF_X(qL0lW#lQE2ed{L} ze@7jD<@`hMyv}bnU=y|=mhn4=zpSseaVKYqKe{i7KjNB%Uc@v0`p`U8{AY^4_syJ- zFk1Y_ivMu&M`i7s4iJ`4%hV5SJUiJ4svE=~d-&Hs@^1I6UzlusVGO(2P4T}h{&&Qm ze}#KmAE*P!gNSoK50gg`_xa3>ORtxnxlbZZ?_2J_k9|;jjXC_a=JnT2kpGcD1L7W^ zO{npk1e(!?pN?cR$8qm{Lccb-m;D_XJK-UHT7CQ8&GP7W`JZgQWBmJp@o%#Ag}Hb1 z_M7Zve%tBZxPQfG=e`+|!cO5ds!y;V@;^(@>_6%~&eQwR#0K6t#XBO-A^7nmdpPFCHuZQnle%m;gIj8O4 z3cus${ej)td^8Yz$0J-#Gc%A>XP2*hka`!(*zH1{e3S&@+@u<UTkn>wx?aSeaxnK*|~{fO?Z+6Tz&zc)$UJ5n87 zp#J|bFHH2D$tc1!RDWds8D`MycdGwTOfNx`y08&(@3I7D^P7Wtn2!Zmge6#p_^oyz zmX|BI~UMg5O-e@^HZ&Lv#K4cx+Q4Ba*sMn)UneR8nK{0Qv>Mqq@U+hU;lZ| z&$6E8jNo@-v^`SC>06X4V@CeRoQAB2IgL~QeNI!>zt5=|`R6&cNB%sgF6(hl{kZ>_ z)0TBNr+xLm$w{Amo>PwItou1Fi-&SfeKeHQy7AxUj1t}$6kLTu4;<$U;i<~9&YGrJkL%EDz zzutd~74%ie%&T6i{9Gt*E0yp2zR0OP^hHkHu`hD!$wbQ+ISsvEvDkH>!V6weW^OT!q3e2hTyL-S9B)xF$v$doX9 zH@|G-MO$;TvcKh`@`r4_sr(_^x#Jx1&h6%Zx4+04C7dycHrPV4`ofExDvYPst6$>W z+sX8_@$n|^IQO=JjC**NeJFh{_J1I*|0{A|6(JrZcBRX!b1KB0Ms@@1d$Wv9HZylUGkZ}%!+lrf## zm0#YYOZm`E_AFN>kiFb#vhNvlfik1DKpyg}C5UzBGI9mt8o+LOq(}Z)#k~e`?O)$= z?VQcpIrMe>HeeIBU>jopp^S`cfy&7}sKAhVCo^C77wz}`{4(u6bE$WaVH+sQwPszK z$MJZ7jea{hu<47O8glTV^g%YG4N0VM8fVdZAS;|FU#&0er(ePqbo;M+q=(dc>Eng+ z`H8%6Pyc3$`fss%0vY@FA@$!ebu1b82|ehy*Kh;3a2t1V9}kh4|Cgt4tuF5&Q~Lb9 zWM=%mRR8~p_JKO}6xq7k`1^L_?`#Dfi1YsjN92Xa!Wnp&8=jJ{+t=#s=iK3+`$oT* ze~?Qbvc_>3xtl$9m-cX_cg^P>fl(NPYISNA3hDJ*#ciX0GJPWAp23aJ%rSUmt%3GR z%>nU89Wv|xy2QUq{2Rodoa{G6n1&fBMhRwP4w9?oKi@Rd&i0Kn`}%Qb_W4&AH{TS0 z;hYj)E14PpsMS6?@^btm?uj%{IP1)QoU#LL^dy?-(FPFr7>zc76u;9ri;2?Pkny|o^nP5zAiK#Ga-bqBTqAGb z7VhFcGW$PoR{m{gKbUQv?L+o}Ed}ABYmf01&k_EGvKzyYj}aJ!F(|}%OvGdqVH$?| zwa@SC)7uAV2KV3?dytSNn2kATy<;s1Szi8Dm`^UitMz|N=yC7K%=*9M`C%D%-vnuM zmNHnlDe;HYhJ*dE-c@+D}0eenV-my0z{Va*5t^RoJ*fzlOe+!7ZB7SiXylG@H;@P()9@*l6)|J;!{zi>`w#D-SD^2@cf0HzNBe$=XTHwhzaIy27}e(LRhj>Hl%CoD|508z zPCtnzby1u{-{77E$`)$>Emi)lWQ)*dsBzD$_4f&SX8qqH<@W>a19fi-ao?WwW&Qt) z+6V4wLlP;R##s!_&JE|ufoW{~FaEZKvYq)`1h<)GN>y10q zX=^;CKSz^sfX3zOe|`zqYP>_Ocd0{MC-9ff-#unt{Y~i|!;p_J=l>Dh$-V3w$c+DM zL!`RY%}6iQ|3~v3_5TC){|oiMHc-Ysa!&n!M*Z);8vjMD|DDiZBiF+};`FtMfIA=_Qzr%=sM3gVZs3Z>K!y-aay}@tx!PJXFco^U2Dq z_M{~jp^@Hn^JSZ&fvkHVKVD|9e{5e7*K4L5e?%SXkvL!uAR5tR-rh3ztiUR)!8&Zf zCTu~n*Zil|=0B}B|A|bghkMC1cOTiTt()0j?V|KqtAE&K4q%lzIw#BlL|L!(9mA#H zvDPjMuMFuW=0lFMp8u%x*Vl~;XO(?>_*cYhslu?IJcz?+eWsm49>+=4AnxrD_jpa; zFedKXcW?E-@^40_{wEJI$9pSV|_m! zazDmXJV(v&ckEAN5BzcOg!<9%ghYY8?~U&_q6y6--!Z@Jop6f%wDrO};_^;tKl6_C z{SNz|wfN_p;m*JNs&7$ien6M`0Z7yPPWX*A`r-eP$1n`}=(sG5Dr?Z`J@L0s?7vSR zgFeqm&k{zdu-sS3Z#*VqGKw&;;+-&!9BeBLGst3;U?_|Wv&lJ_hxw?S`F2=9F2WKl zL$z`Bs!8T0(Cdw}*KKAeuF(hNZmKdjpv%~KuQ7D@m3{OV{omFL*2uc2-FNMvucPle zuAP)C4X6`(8vN(&{(Ij(O~$<#YL3YNs4I~F^ZX~{{5ND~{pn2Qyf8Ll3$~#Q<=BG? zBqz!LQ-wcE_()9=KGHX(|EtpfNAkaWPPwO*Y%7&E$&Lx~zh~_i#z7oLJVWXzS#?Cd zrXQ!DL=EcX^?G?X@kG7(;$`2UDY{*2MjK-LH%X>&8fTHQ7fN5Bj+2hL`*8`o+v5J; zxf%Q4HSQa@g`vCV!IO7!AA`@$fhS}7e@s5bbA*2>&7gIqv3)WhBQOfD*8hy5$3A~I z3hCp~H=t}#CiEhXe>ZojpK)rq?}N6nzE7U-gHGQnNoLMhTkrd>_I+3SzIe61a_6$I zgo(nPj3SJZXQq)eP>h=C>=(0?dl!5^B(D2@XrwnG?%6X?_I4<7?_kPV1ol{%9d+p@ zJt8;EqtC}szH~}1!V)Ztew*Gl|IPZpSOyN1a)MasB+{Rx0#1`}Ogs}$eumPK}1=~=Dq&l*zLH$={{Cl|i zeXRN&nf-tA)bE$oe>jC!{POvLaZkW<;p{;L_M>CC{%;oB1HEUouyL4v6n$IOtsB*` zdFomm=QnUXH=HB~muZg__!cvzw^FtZehDs+GbkMJm*U(OX zW&gXueG9*0|I;6@MP{91sqa6@_n+$ff9U&vpl#zfcX1yN@fc6>9O0Jl|Dp7WxVKYg z{n=LO6KUxu+JY^MOI@J{j&*wmNSN|E0BKkq~CGUFWJc5 zME<~-+V@u*TbnyF96a{*@cG8Chwm*OZGXjole2&6pXK~y_vOU-+lYj2wsAruOKaQ=H?JXzV29e#S%eu(W~3CH(JJ6Xk{dH8$UmhXmB z``$Hn@LQoRYofJ3cF@~BkaET zjZl8PAQbuiwr4G-(qIYy*_eYN zV@~tPxc+ZG8TbEQKrX@(EJHl|a|O8yYp@O-$Mm0$WQPs(p1t}{z4{3BEy(O&won^x zskYn!?f)qSVH^K4lw%L7ms>}$v>;T_>pwI$yj_2rei%(h?EQAY9&lu0VZmQ+|4%iS z+CA-L#|7)(>6!NbFl8`z&pGyiGwcIMqYo3^a}Xy{g9MtAFwsFqHV3Y|L7QJ{lRGK58T%^lYIg`d7jU|_mQ#{eW+ZiAMH8i zXS2d-@+_)%N+3Zxxju*~EOhyqp)b;I$^r`7_&RyL5eg?f5 zne*?au)!g&0ZRAhhDv#?#I@O|KBGTC&O_B@b&1Vw_-_|~vX8r2 zd|RG~KU&X;zqq%HcSpPU>kkZU(FWJAj_vHh68&u9uS0rNZrDJUC$hs9a`zVY`TV@F zjb4T!d-V*>)Bk>?4(Hy33hYPb4Bi3jqUdo?xbC&)WYEix=Z3yn^7&F_v_3*?OB{{< z6}jPfl%Y7BBx{gBGwP(jm)Ol&+YRqRUaU7g%* zWcy~eqvg>qRS=S7*WzfGDq#07(Do?^y-x~4JpXl*=d>Y-6spZdj&;CkdR)6!H^n-8 z`gz1N0UF1uqeiN$)LZ@hF5wEU;mh{>8{D^W8=3t#*$eN|@8cmJqo(Q?=lk{Qe_v3i zURSrGk=}GhJ3>3?sr#NI{3~rf3`0Ifz*_9kb=Cim9%($b>3R?O|Bw0q$!2wV%WCWY z^!Ho0>ub^5*BgHzJD*4sGtILT&KMM8JUW(Ye=Ib2n%;9l9sZGV0D2MnYPCQ3^^&P} z`~L8ofuVn^hM)7K!{?xPLAfsD3_b^Ka;^%KZV=uoF`A-AEO-o046V0l(3<1WW5 zyFlE7zL$NVW~25g>QIkgIsYb66889A1@_}04&x|}Be~Z28oGzG&m%QX*^sAPC}961 z|I+z4?t8sIAH7q#a8ejGNT5>RqM59kuK$5HdJ-wbvjO560SWDnxNrBX{dtGRX#bA( zjpg-di$Cq&^Vs#YM7e2h0sRuL;2H+keI?{dO9N}Hz2*L^?0>i1a~pSY9}n>uPw^bd z&DvkveP^=gqCBk}@4YFHU(o(K!~WC$U$FoAUphZh_E_5&2EEr~^FYb+F{NP`nU4_| zg)u0^cud4(3|XsCMDExw-Wx-hMvrq`ipi3AUA>WBWbcLjSz$K!pgjcVkn=DfyRP{@ zkFA+MRvONV#|(Z8un5&xOWFTQ!xDP^Gi$RSm4;>X6=)i(T{ZH((BPf~R`F{+@m^R% zu0uz|dtn2)3GFS``hRF$I<|0cLz>=)6nas!_`OiO`aN?L-V60)V!N={dPbM^?bmvsXrs5I!#$lF-_N9- z$^p-5QKz8#!TaI3aMSvmRgdDjeD8(w+s0z~H`1GsKm%HDmV_GqnfV3AYZCMh>*Cv= zl!Rt_kG1pNE!L9LljvLO-4=RB_oPs>Q2ddJ|4QMa0k6;hIxbE5-D#Y~dGzBFt{|x{ z>_V(#dXSRmGwpwRADUl$sr|3MZ0pj$MF+i8+q3mfX}Bhw8@Pqr=oqj@-rD`U^mrz3 z_pI!2pZ*Yi+u8omyH)#fqO_w8mg-c^#TUS{*;9(Zc)D>)t$F&Raeh8ZYE-2bP9 ze6|12Z2BC`!+-1kKXL!T`F_6ui?9UC5ZC^#Apd{w|1$~aC5$;Ia7>0a|z~H9bFp7+O;*B8-F&+~!8AX_e87Rh3 zTW%;JXQM`+|MmQfTIGkl*?>lR6XN{+IqsW>`B;EOSb}9pu9yGy6T0;wdyt~{7AX6t z%l~M;F8||{`&w7a|LUrCvICv!pdGzAY~aS;jpalec-)(Qg>cjIUxzw)4SgMY)Vb1t(*`dnyUn}=%Px9OKlszS`*LcKMa8F*>9!`+4EQou8ZE?>w zl%X7ZP=P_;dOsQ0;T-}7yr`9?_7`_Sxq3r->KmDo1&<=Ez{{r|5C?*?w+Htym+9%AT< zJ)p?g|9?sjp0qbY{^;AL*A|)i1fJpF(zC+t$~1$qM-5!X5xk9wW{67FwCd5QdOU&kapInMX@eYz+4{`chn zXR%%HI=v6gANu|aeSfr~ZKk*hw}b5TyUM-#g2LHlY&@R-ahhK3Op+@8XX*8a*njrf zBaq&Yrf2Lw)`>LiWdFe>eyzvk{dw}$?A*|?I44};cMX~SLpJIcm=wCdOhc61}oZsE_J2K7PN8T3ZUEIe*?A$gs zJSLxFm9$$kLH?HS;vQJ9?U#@AAJBy7?hF5~yo6yGT428^as)M4S?YCdjJ6hI-wnpV$;t!T7!}4RCdutt$blJo;WjzwOvk(AeLTctRE~50-qe?Eh>U%YU-eb* zug&qCUjNAZ-_yst>wlQ6PCw(j-t_(h-k<+}cKw0>qEWa_WcohapWkNIKaQ6+=_4=- zW02f#{KbFRUEn_;Q@(NUaKD>n{{b{>54CLe?cLW(wk`D^%=PX4znx^w0r}&IGJu`o~)SO=W3N$H8V%~2ctzqY`_-mUSobh;kb~Lr@HoOe~~@%R*L?5e0MAR(?abpyx#xu zk@god=da%}hyR+kg?evCmHH3cgk6Sm44t*F13ADpP(cnBTO(k8!2akbEgd8eBQxL6 zw<(`psEua~m)~)mMBi$0K<`!WhF9x>YFta8(i!5L$Y!)5iE3lwRaMR+r`L}*{*BZ0 zvuJ96J2avp&-gda^XtbYT)~&e->z}rz%8^ZFE@VEd$;+<*R+ASH*?n*|F0eWrTGQZ z%`Y&v-)K%j(}Z`}NB-jet-tj9?cDZQ30>FO|Jb;DkUGHrhcq(#Ukjr}n5WQ+w&CU$ z;MMthned%+Gqh-2@F~KM_>7ciUh-8{3mXi6X#oXS8^B)bPELzZ(v$pAz=t2hT=^?>`zD4(%+mc5SM;0Bqm~+0Y*SR`}Mw z-wx%6emm?OC)!|o&pqTlzPasOt>&j&k4FuZHjBhwx3!v%iBb_xFK!*@4qp;UCkJ>%SWQguH@( zj+?jz%MQyX3+G?cyXXTa-wwaSJ-Eoe3={q^^xK1N$Td@9%l;jnKv$})zsz`o^#gxS z|K2F~tgwbLWQQ-fzv}nNM_&yE*BtJ>`Tg-5q3mDc-=UEEKauZ{U;Robn~E{`5BOcw@_(0XBL91`T{-vn$j*(w zo&ER87Pbgmrj@l8Ov&Cy_TCHOIQgH@fUdE>ll>3Lo~5I*-x(Il{;qc@A%EAm{2$3# ziozY_M)Dt#zeA39Pbs;K{M+RJM9y{Xf5nH~jy)**JLFdWOUd8#ZvQjcQ}CPFm+1YY zen0yi?!ML1ihS0~R(!`hl&#gj+id)GV^;Xq%?a7%Yrhh9?rpR;!ybkCuMZ^&a=;RpKNhc9P^BkJ8B zs(*i^{5`t%4g05#%Kou?euCqhM`fSrdL#VwLu+w%PR%|!f!%Q`J0$;~wX2^`jLfcG zI4V1yPt-C&JW?~WFP6;A_WEUYy6y=y#V)*SXYnuauW-Tjw~#!cZQ#G`gsP+LPh8V9 zExVcLKjqh*r;NHPJ>0Vg%?o?aTu`^(`9^lCKwX=q-X5^W6`J+yTg(;hu>P~ty=_nI zLG!@6&{A#3GtSW?YiHUw@}fB=t|iEZb9v#X{44YF!g0?yG0VJyae3iq+$YJZ(e}IX z4nOky_sGTg@y)#Olj*s}$+N?;t9jw?^V^G)-g(+@#;M@yHu`7&yMK#+hyDKZZ2$io z_`dXikh@7<`~%m%gCF3hsEfGv=hE4)za@R6Klg8DUu>S1-9MhZMUKE4ESZ*lu_Efz zvoDUCp51>X%HPlK-!MM=;-$Zy-9L>!k$-+k_Qf^tXZMeJKl|d0C`+?1p8mb;{y8P| zS=s%?zn6WnhIc0oaANkw z5fiif%m0?&|1H<~r}!t_^VIWi6=z@ERGj_S#o<5rJ}ag1J-9gjxBd2avg`O2`Ck1e zz3*}N7ynlFALnuxzbpL+Z^W$Zi!0vC?)Nd%rN#mu6lMQ2*L#$yxxP_X>2GHLsW1%1 zl~wUK4WjH6`3KUC=7{s2L)m6}Q?K*vj{J@6hN}GV54h{k{bqKK>t|j6Yvd)8S*&ad zeK{(##$+Go_f6rB!%vpyhaWG@H$Nml{B`ahEuE0vM7E#}N&Ghc1efsn)Zfbfr{p^S z^&jA~y!`Opjrn2U>in>k`#W>TXRq-7duRS`_WwfuJ#6E@f&4=-b(IC*WrydMtwfOj ztt0w0-`hRE$h?8*HrCf_en*@69e>K*G2HwQ z#Ph?G3d`hN-=F=J{_PT!G^h@M8nEN^8U5eE@38;D?=~&+N914q=1JB8{PG;(j^Co*_nWE6kAHBB`p9o44IKM; z7rz^M>z}h9Ne*Q!7b-){=lJJ#M)!z>Z{nMXKsP3aIk-j2H+nk06y|y2SF;uzDsFJW zC}@1EB%8i~Hatjjj;*5q6)d_L^rEI#q5p*aA-;e57KNO*T@<=fio;flf%CdchPJ5E z`BhSuwvxW(O8%+4QYzD_uh_p(9l26!bX<%1IAdpH;!3Hfuh+o!ja+Zy`r~)ww-z1K zeXGPmU#SH-8oWZReL=i0>%Yj+Q1tV&vGd(@7&#+6mHr~n;dI~^jebDHUjQ^y)zM2O@WL--GHh!o6*h(0=6DeA1N;3i0dhu~*W=3FJ^yModG3YN;f!BPd) zm}~Y0OKn)N)W!SXKUnIK4WaC*2uBx=wyz1<974P32ocLN?c^fuWB~0VZKI8Fc4QmZ zwIdzaJCL0bw5$3M-rs*dGyei!4w2q~5b4X}eB}(WwF*vA?oIopE8@+|*R?mHyU@jh0}w!7)$VE!8v9So8g+T3*kv@drD z$<@dWuo)PFYHkF&mzsWIE$1u#xrmuzm}dB0)_2aXkoPcu1n##<|3>Cr3Yd3k=f6(4 z4IDWnt;d685%ZUjBSG>{q>l9z+Xm`m;wrc=NJ5Zn;Bx4R<=I|DM`R;&J~1nVX+RBI zM`;Hb!|q!l?;s;-XZ{NwgWo_2yhgZ>A(*h&09_1?mv__7+@NyiZ%~?L%E8hN>O#Jg z_qnsH7zgp)lio#80D36Vg~)fgws@9uU4*A1#NX^Sc|-k{2lUMw}7`%#;*MO7UQ zmFmgOsvdj8^!2I^Y0O+9k8tf%(71fDWFgDob1?I6R%nO0pD@-#_5vzm=o(QD!e}=z zd*OL_5nh7hZ~{)kDfkP#4e!DSK$l;07CwV7;45I*pwTuSk|3mc@gZfIen_<}JfvC| zGsJp2gR^fBh`suNwD}I8AACR@FhV0<4{3co4O*B>Di+)zTLyxN8$59q%$OY!_+F`PauCCJj5!%}8D zEalF_Qh}`W9hNF&HTIhHLsA=iNb155iIMlLUpOQU(71GnvH2ls!rVN6NX(Ik#1ef- zTA(%YkXVs6uCpWCQV&V{zC+?5t`1};>Dq;KV(&)w;NKg4i2WC&KXQQU2a!Xh^Ki;< zWMnEsMwc_iwU8n1r3~>*XNniI5BIUj3>lx!pdcKO?8F12-E}~6b{>%2qywT$J|KBZ z&q#jO0meVfYhW)-IlzC=pOa$38BEVfNgwC-L_Q~F$#3N#W)0*$^IOUH{#Lp$kHYit z5}bnf;S5~0m9YhEfr?)qlpBz@K>2+K+9njx(PzLTi$XYJ* zDeyA90)xLeC__O9DG)0p+q6Qo$Q%c{@GknlkVnTq9|{&$h#m@;82>^sW&@P4hF%I~ zbkxhCg7Ir5RHX#)-wXbOfw>myG8z9uJ>~{z#Lomxn47`uLf0EwjxqiP>-5)qm+Y9^ zpnZuo3h2Px3I07xPUyzm1HCThSD}9w>)SAx!u%=>WA^W_{LlVO%pUOK=YuiKhS%XucnjWv&*92jX?q|F z?t(lhgbr}S3-AtH6wka0JP1F9U%_+088DhWD1;WU!wc{-yb78t*it#f*2@P*4>u+TY zsLJ_YO9ktUl~9$khWuYc{?j*xy394?{~Gdt4f)SGUHF-x33D@;U7RBZEyq}k1nV@q z*I>uo2JK6%k3t9LPUynV3Eh}`px4FvCiL%OeG>*#Sl@(U%p)+G$@(a`F?+y^pAW_` zkNfSJLE?ikeUOZs=vQV0q0bk@dPb0VU~qqs48bsrKri$`KMX(@VVuy7xd+;lg2VwG z&hR;Yw)i{ zmSHasroS0Ve-l|8M|p%2D1`#hLm?DFF6bZ+@<9XHpoJW`_M3dua2xyz4!|MMLLT(P zFpR;g@H%`7U&48a+Af;!z=QBGJOPDJ0!{D&yaI2*=WrfW0^c~Sg3I7ah=Om!ci=8~ z0-l7Y;1Fmb4@#f{YM>sPpc{S$ZN=C1jQg00ZhhT2{8P36D+&9O(u=W@1tLHi+{iw0;;!a;m`*ZHaxZ8vK z8OG>-ooa0^I%JF?=jb5L(K#H)Z;pK$zw=AR$$#)m*5Ur>zlJ&dKlvs0Gk36N^GY#UOOdHty zb$BE8v0U67%=H#4d6FH#AP9AhmFt&q`z3Qe zR|iRx4l`@k{lq&=|7@&_@n1}cTsWVRHC}BoW55rBM9W;zueitPS*~F{*qF{9t-Z)h z){TkVZ}#7B>`v~9e-jgT7rs~9zx;jJuf~5p{;aEv-ACL|KSNk(K6;VNt;WrZ{}TQ` zVZGur{LkUf`b;x4O(UlW7ett7=EpY@=d+j-zjo7pm+-jz-|b8z&lF7f@QcL1KeRrB zXT`4NTEG7sG~w19O1uGIKjYl3#D(2|X0^XeoH>ghZ2KnxrxL+=5&~S7G>55L_ zLEMA&_Q*G^z2jQ@VdnH9Vz*9iCiIoCOf!^L%&{bBs??$3Lp zgKHk;`WFdfB)mT^z6Wg;b_05xm^~t-J35rCCpGOyZvwmaA?DtLD8IxvKp8XkG57yV z+GyT)WIN$3tZChfKhM(19G2gH=5hc1&>2`uIG)|Pp6hfxR~l)+Gx+n=|Bn8;z;5Ol z%LtF(9N)A54E#9;oO28$=NL3zs80->Nx6;ZJ^uAIM$URX!*ic`i1PnS*bLVZmN=Vy zgyo$t(EJAtK@pz3+~{Wjl%aO z@*MUv*U|UkK4)$utqZt+2KNsasi0o0K0&(2VxJT#A?(;o=r8`9vp%WkwD>nA5l;+y z0r+Wizs`rTmE0HmMc47%yj#<0(%=N~?*BSoe>wC2m)XcOoFOmQ6RwAER2zhD;eA(2 zuUySJsO)jV+>}cEd%52Gb@(N|r~TB0{=CaT`tG<|v?!{KANhu4`s>KS$glH@J@#{F zBQLZQS-4-wSJLl7{_);=zneb~`5ydvwBWB#|HAY0JuO{9JaHh}O|3$X|T8^O0GQ)lt`qVc1)^=pu zGWx0XxgF3k$vii7#i6?b-OvNQyV0M6e(?Vbb8tWUH82b#FuEK49B_jN{{7#sf9v7- zNP{DA6pq0JOu`gQ!wk&A*$qnOkqdAd&cPxyF5V`l<=bQlb9tGPWuyak!Y;ogkKJjf zWEV0SM*D6P7wpEo2T~vvb`y3lav$u$oC2w^57HnV_Cp3_LKb9$4)kDva;OG>oohs< z5!YGF=|~fH3)mn4doYwgUp6d4BL5CM^3V-9H#^8*g#5pY5cW4SXg z?NCFQXRH3EX6s&6b5(y=b8COEW~+wPY?(>T<+iA~mH8@y`$fW;(E)X4dyz76&D+Fx zW<#wy^Y+_H`>v8S>?@gzb03|SHAsrP=1L&RHP^z8a5HR&Z^L)t9(Vv|HP5QE^d0Bc zKc%#ql`?C-M9r>T$Nkr+xht+w4-@9rK<;zSD1CeW|(&8O^iB09`cA zgAj{34ro?1Prw1tLIIRO71RUOndT=DPuK)V$NV_=O2nK5RF4{}1`XAa<~gW?C!qqK z2C4zgNzkwco}Eg&yodfT?NM$fIvJ3MIUfpAE)u?{$qF> zDniyvCHuCk&awUuHBbw6leB?QpRisUkd3VCo1iI+wmxz_bv1Jo(2&R+1(+~5!baY~ z-=8a|KT|e&kyJ)g#-SR&DyWTQjsx`Pc*cc`qzGBKbdebP(AC0T>bOWsT)Z3Zn$x@+ zQj5QCl6S;EADK7pKRdyHh8ktlM;QelW-p9W7mPt4=3W@UuOEgm55kCvG7kQK2e$Vy zH8^ zBENtr%&VXYvj&ENCQWl5@_?pL(}AShp(#dQhP)a=_@+N%F6?FIGjdnZwgpqJU^CnT z-+_DK?2Y_G7MTh^g$kIv`{}?2vrZo?qjUChjHuAxo$sY z@;Szr#f(9ld9HSn+|9jSWSwS$`@hb7*I)UrJ?O1;^DH*eEKv^i%q3co^NWD{kfK89ZkY{z^fl$$S- zPq;S1ABH$H@S7lgdU==GlRE-uAH6bg?k`&d`Ttko?D|gwXYYJ1aPH0z;dg-pgc*i2 z&wdnm=J8JhU6|+n_V2zGc&08oaCWdLaQ2Scz`27hfpc}3wgC(Lo;ThxzBOp#8f< z^rx?s!uhMDXk(-lCq#+?b4l7-DV@4f%5+yrc{1mn@4Q4RS+lJ|R%5S0)@Jb!KBN(Q zJ+dL~D*k(WiI|`Xn!$YJ60txFw1U-jvDm=wxLDdIIG=x!{|_#+{~y_jy=$8N|ImGm z{r}MGV*fw%r?LMZ2KTc6ABGcH|Ao;o_Wy&Me&GL2KmIe);_vV!EQ3&7H^6KV-#hl# zU=8+L;a>PT9EKX8cxgU|O9*=@-+0nW>Sx%2Ibk#NDbNZpf$LhO|N=29rbuN*4qFIC3U|B0vn zLtS1&onDKqOJ}T>@h|-+WFvM{JmWORdCf@k1bTW?=s!`r8Yr>Wzp zH~o9iJE1E9-D$Y+ACE-p#+}rW$zPu>I*>};i2XnR7s;Y--9`TgF6{ryLO*&R`hq-z z{|r?P@9zlb8w=pu;v4M%C*Nro^jO&EfTEiv<6<%Iy;xdLUo0(rleYPb#R_eFTlSQd z;v(EAgA@k5@P;vU1^hrM??l)Vk1(vN)zIf(xVav1w|!rcu&gnsZsIauHX zWSr4AZc|@GbZO{EoWdy&&3#VAg|R-9^*`@Br}1X>y?f<_O17Qxi_u>)=~(KkS8P zARp>Mh~pBt24dk3`0+~0EgT^pmfJOvA^!Seo!o-F9`{&?^M}FjFSI=`Z=)YiTjN3Q zq+ig;IHuY&6d69R7&?qm2fZm3~s!U>m$+=Kil z;qA!s&Eyqf-obqfG{U10&h?++K96}lG{XmQIUIu)_!zE&<6wus!!n>MpeaPAAOiul z1x*s{g4^H<_=31UBTOvj)$lI%SFx8sD%Who?GpF^_bJ@U;YYYTkv~Sx-F+#2XZT{B z%#Fgwn9cADc=l2`^L!|C2Jj5#2cRAP0;Ktw4eNxZFb%)w(X1gpR9iHsFr%iW*@$@s zem_I5Mjk-kiu@*YU>*ikSr~V$<$nL-nK0%|Ft3Dn;j^`J#tU!5ao7%P;g|p7KEmYk z%pbru%=pdy2=kp_g+ZS2CGg8SI0%o!eXt%rSWDe%tQyFEUZ` zkp=ueT_2yw_%~6Ckj2;yxR>BxiY!Y{l=6&3smMx{N@NxGYTRq^uSM1o-WZ=q-#Sqm zkd4?)xHsY7j5ME46w6|wv@9n|E7FSHhPxgAHe@^D9r5fV=lV`$7j`G^-T3z)d(-*9 zdj|h^&*J~?$U*EwxDSUV$_R|cC5j8&-~n$K|DA_17zYjichBbCwY+;y=nlyZ-yyol z9g>I4$6gS>L-dI|q!3wzy%=``{w2s#>}44{q@1$^Dv*`ft8lN5*da9`JERus5_X6Y z>M=J!BYq}m!rTnzr5$2{7R;?+#m@$I%x%z~utOZsfw>d9@N+^p<{s$H-6@7o? z2=>to##mYCKqEcay}0}EA485~?;`J7+N(N?zM`k{`+B2hxn) z5=?%P*R4peiS)FQo?JhU9L4U^k=_Q<8#y>be&9Zg{|K@dd*4y=Yl8ej7Vo=V4C%K^ z39>Znb}86-JNg#4OChoddoJ!e{PU3c*fqho3;LR(Mdn~{Ox-S~dA_I9d{4+a?8cqj zSwGw^4ah3jcGeHKvwpap^~3E_#&zXfUxBO)NRa2LgHFL)a25_wKSfbrG9|5Hn5=mS zvkCb}@WG4JZGWL{`YmnID-eQxE8GP?hUcIh-k^Ou3n7G!f_S(aeh5!PHdMivfEt|U z3fKa-!A`gz9)TZ08dO6Qr1Fdpuc1wV+ewbDRkSe|a~Yw&vk_bS@r=dkBtzmj+^=2^yw zdr)(w4CO}1aKU>rQvB|Px!k@g{ue4&$Ux&&=<9wgKI<9Qbv}`C>uDL4x5dR_B5n@G z@o-p^^?4DStQPxAte3M1;vM2&!8-(VeY z{~Px{!hFL0in!)d;_)kdW`j0itK=**{+Va|bDHrFGC%7kDM-3W^c$5FX0rb&ntuzj zm(YML!L2lsHH>}eKQnJ!fvn6zzZ+SNy#`r}KkEF9y_uv8U5!B_9tec|H>RrC;Xr1L;UA_2gbJ-{x6joCI8<4J8~NR zmnrrq&agib`JeM=nG+f#u5qxCJ_hpPJZbe)()8P;qnq@3557SfU4z>WWH4q%+*k{#2g{iVKzcNehtuw*#u24^q;`Ii#(`i zZf+-YbUcFG&LJeLM%smwhA_nZ#UPBpFu1}=L-2qb{Qoxeg8$!!V_;!iZ7$~= zJx3_}L^(IX6^agfsI+0XBOUm+<9^{k#7Ji*b@G1JZqk^aMV3b}J`ZDj9?SR~S)IuE z9BQ%GfH9KsJv5{=lHvcuZQ@0dA7a`^dGaR|B-{(hj1Upe*`&7cvn32Ki7MZ-szjf z$NP+h4a%}!3-9#L)#MV9MT5$?1gUNo^?p0?iy**Un9-X zh~0#}BAETdi|nHcp{y7Zq^9p0#+8gSSR=0vxLOK*%u6OQPpM;`(!{(Z=F(lvQyxnY z1M8jIaP}M1&T4{{WFzw?XxIB_=dtUKQtp?~mtjJ@57|4*o(UAnhr_4`BB%$jd*Uhg z;EJXm0RKP7$H4y&axb(pY1{(-Jsf6eGqHCAY~a`Z>tw>dqmMlt(Obm-FU=k!dpF91 z#fj{)pu-n{E+6+U&q$E6MRfeI*F>Q67l!U1vVnaX^$EdZj6*jN*&K>%j@9@coOW;0gCeYCR{`LybW?FTV)?o&bih_Tj8UOX8Fr0?1OUHNLfvUFv{mf zNCYFi3uj?9W%g>gn7G!#f5DHT5;~y)df*IvkMjEi_#xy#2xU1Eo}m0a40&)F@Ai&g zQqG=)L(l@hhnL|)&=@HH)s%k|eQ=Bh<%UEe*uin0U3A9F%|JYXSIe z|M$`UXVLy6ZP@MSBBhObrrpB)H~jzpe@8ssPTCL~^WS?Tq>nlB{%OiT{no)nzFqpU z!^jcrqem(KlYGN_BiZN6H%mBQ;^i`Sly7-5K)xgm){suC{NDi#!+UTI`FKCv%s22D zyaJbz?@{m_xCS-@@FbK#D9l_&e!v2pfjRgX-iCFTlg_Y+`BPvBsL`O8mJR>FoWr_$ z?hxmr5+<(?ePJjlN1qr9p$Ll4vHlJvPzq(`=>J!v|IZ{rCGk}0Sbt~YpayC&*ST1K zXX2oqcpLgyf9D=1^y2*baLrs}-n&99`&Mu^JO2VsWB*?U`_J~T5IeMCZr{cEz{#A! zzmxNUli2f>%HFRO&IiW52lw7M&i9SyeBVU&pCxeiPc-NIZsdGH7{NRmfo@$SXA_09 z|16CCXQAvr3qcp}96Cg_E1D4UKa}zxN&ZKW|I;Em^o#PQIFAS&t^&g830Jr*?0?{V z#RbmspzSG{XHV-a?UIT8zeaSwoap~L(ElBxUFs9sDUn*tb$WEr4Cry`L>jU=|AO!) z!Z+dGjJr8Q#Im2gPFd`4%@nbwJ)msq4=DRy_PL_>)Skkg&s4_1dqg^SbIwRI|Hs+I z|8Wv2{|S`;m;sD9KQt@gJoy-kTm=y@{rH3G8^~ygg6rT0xD{@KRtxvG@eJiW zOEvf8ne9Bg&BwF2xF^rp!LvGe_IBQ(2f8skaqGfwU>UuwCC=DBhfdofdRs>HwoK@8 z!60Uj6TK}5dR*LN6fT@cL%rtz_qaEWGd+2RN8nDNTGHGN_rvWledBI*FY+PCT37UYev4r)_Kd(x^JS(Qqf?MF+-5N$NhR|=o=&lN&imt{ZrWwfE>a;jQj}yb_;F$=_KiJ($?E((|xq< z$ZqUCNV9{!L3t8uhj)tq{|Z+4_x}~zF#nS{uHd;Jg;Zc!U1Qo4EQgWJDYVtdN@()S z;D7B0K-PrPS3r*9{zo_i8^fqW;U-9g`#`gwz5!?<2Xc4QH-Nmm|AwEyFW_0I zfLdq$GV24EEa2q)lUxG0>pO1Kfe3%lT9_zj?csTqXl;V4YNC$N(5=5n|d5+Dig z0)7v#c>**}%0KO?)<^kA=3>`PQUA|S|04@d)BZ0~|1Z=2Ba5*ca4*5X6j?_2@*(Pf zuCGK^VXxkU{$DEkf5^IY^#5ox>a)=QLpEYJ;ogLQGt%s23^|1UpO3yI(u&2+5ZyE{udYRKa3`@ z|Ha_%!?OP+jQuXzlsy>Br2h}Oq(K=pLMOZg6YvT7|4%rGnc}V~0;&g1Gf>#!T=`5w7{15%R~ zB-xc!W%?j=41(zY21zAU z^##%X1u_4{`j0b6>Z*gp$YOPU8tXsMn92GNG@<|2%pPcSDC<9w=vG9q{<9O^ieT1% zlF|Q(XZ>f0Jq}^4|3C+H#4Hcvk{U6*bP5!bgJG-9tuHsIfgG+}Q-HtT3}vuJy>`QD%v ztY8B>v_U)l7Ke`S&A|7D>@x8^TKFEJ2YNH;|3E+er2!rL|0jZFXp#N@%Zz1@QT~_s z-oQ;;iu&(7_1}5wzw^|8=f#+IUg|T?GnPFsjhIc)lyY90!E8G(7Uy~T zAm?cx&(l7h7aL|fZf(eR<9X)L&P&I_dFecTUb+^~i}T!h&hR_W`JLyb*Kl6?s?SS5 zVFqAOe_r^%JoVps8S$N$QQdj!zw?~WcV0ZE^WsJNvd_yHjDvlee*p#4?>w>+-AKmr zoF~x=X2#Q&p_S4dKtB_@=w~{iBawb+H2qSfBc6U{7X3`Fzwlpce?A=J9EN@LOUaKu zxbPoruJ>}S58U)iJuu4kF6?>qS@R(`mVJ6iZ7AO%@oK{OE{V5_pnD87Yk@W3G4Kz?5V-++hV`;ZF9;2V4g`N-db6*?giw+G-^I1G8v2446b zprWAp9L~Uf)XQ0r3l~$zN5G}Pk#d@7WE>oX9H@jJLK-xK9iD`HX%8NRzhV9orr~Ya zgxl573s3W2eb51K!5gracME}1cmg!Eh1sBm9LP1%{_AM}^|b#`K%1_oZ7)O?8EF5j zY5xgRf-J>eKpfxX{VSGf{}*ZhPt*S6S3{UuWSxWYAJn^;|AR)%Ci)0XHpc&iG4Esk zKmGqQ{>Qy-hCcT^eQz7(-%0sLcKIm($p4K0XJ~KcX>X{{2atorKeWexzU05g|5KEI z!nlzh;_y!VYyO{j#)kss1kYXq-z8uEN&0Um4Xa5zJ&b@0dlBvdm|rH$iS6QQV$Hew%pT$88Bip6MGN1He!c4ipp<8_~sx9_cn+Jru8+Euo0pm24W!&{>(Lh zg-PPnB$NNU$$!YfoSV%0-$v^Ho#cO3fV@GNmO$PQX1L}x>{H<5Su&9;crU6yO+J*v zYU~ZjE08wiW@Ha?8`6c0NB)Vv&K}aC4jSMF-uYoL!*TckK7zkNC~2`4eho)pn?Jpv z7Mh`#@2HROsUPY856mEP2>UQ{B$fH!ea!!)GtZ0k{|nrU{4>|x&U2)JCfFs}v*@Nn z4&*{R?=}GW3+T2_qkq5DCxwfB=(zXM|LqgQqQCt|$9);yc>K$u0{=$htI}7f>T~FS z5T+LD@H0;JOZ`m0G|+EobkY7p6Xxb@^gk?&|Dk1LzqHz@hiMaSj4SO)12RQ??X;DS zYU*7R^)B*$zMYrhHq!0ed^f$z)U!J3UBdkw`=F6}6FE%S5!^Q4z-;0jP>a|7D8z=e=uZcz@E<2=%Aw|B#-Iq=|`iYa)%BF`Gl^|3C}@B?{|DSWt0#&6Pb&Q%%wxFu^40gXa3$1!TL9{GmLX_LfPAx$zH$=_Qr;v zmEN?o(uaFL@eB}dka>OT1sRTIE+U%oeFSq4vwZ)j`Tpnm{+9ye6yMbhj4$S?HrB!3 z!Tdh_iSjuO1C;eyK8-Bulz&7~CQs_r@wavApO~{_D8rj+TfhKb(#8j4yz{uxi!N}m zxWKKazXgM&!O&En4A1n*2w{5r`lXMw>px0Afplvoju+u~*HI_IIViYRG&=5+1~K@> z!t_tfivJL3rXS!NMRN3zrV#SsW9q>(Jlki;x_zW~SigLQd3u{!RV9?G>SXd`C;5>| zex;FLP?17@fo9n&*;9>5yU!~*N4=6;?iHQMD|t4rF=tA$> z3Eh}`pf?5mf77J&S5MOaeO(6oCS?eQr%%cVj4qty{F{^FUOFir@M8ACnCqmBgJ##1 zWP>(kN^&3VvAOiN$*G~fTU3^?Cp{6Ee6zsdUl zUu0xvMn>mn#I-mh?&TTU{~7V}^FN<$27R>|ImA;s>^mj3>8GSB;}m)@r=&jh6#u_mmKE%0t54ffLRFE`$_Gc--0==7C|173wf@Ca` zdA}dg7iRx^F&oAVT|u(Ub-u+y)xL3D9I@ll0iB8C)UV^>+%qoSyT_Sd9v5@OO0h5* z+CsS2xB;;e#uhy;_Rs-oLyj&Ch->1HoEI`69#@EX>1+EqRCg?CHS4IH&z3Mw`{YAE zbDZ^=Kcb&G&iu@{44)og+%V4g(#QDHCxs{^7e)J|7=>$tkMs4S$D~xhTGRq{PH2c| zZT|TH&LzuUDLTkAaz3SVwbbqPi4p44Ls(xnsYVtCOrc)RDqw%j{tzh(ACvO1W>pd2 ztSUpBRaMBCR71^vpVWeOG)QV`cU>=%hL57lbRW7*Z1Bn@jd~L~>yrDg0IDhpL!QOo z9mbe%GwJv)b|K25yEOp!_zQ>+~eYi(~|DvjvCHPfkTLhs#pHbUw?yGk0qxLoRGJ>w(#;h^F9x`=;b6@7}; zj15AWYl5fX0H8LZIS7YA3*}G?^({P$M z=%1`*j`>aGsN5*7;;rH?xJlf!J9&+l%kgKahwEZR_xR;9g4@ebAm}07xL!^Np({hR z{a3h>=KD@~1fGE+*v_lm0e8Z8;cmDOtn~dJK(=JC&PX57O#j6K-u-Uzl@H4pa@;m7 zqvuA&mEjh*@fGnP2lu*VDBUf?nQrEjRO9ZuGHU5ySkjltk34Qj1#( zK%X=$g~+0eVaW{`WjyDWytG%C&vMh>aZ5JOp>?<=$A&$d{%@#9wBaL?6W}4dM|2xU zBoCRN=#hdDkLWQMMtGQS^GNZ2j~KE%Qj+G8(tRE&v$&)@+9MUm{v?&is*N7%{}HJ{ z)-H}n-NK*5xO0T@UxR8Oo<^4k{SOcQKaVsY@rXIhL;tTzwe0dptD#m|xu0#QR@oCg z?9m)y{ii`Wc)pIK#BmG!T7`61t}&S#Z86YXDk zfhsWRl-^=x{o5)<)mACqSfC7X1*!yDdfH0+XP5G1t5oc1m&zooR6+ILcBw(uW?7|f zmsO0I>r<@Ku*WKmaaJ)UTBRw*D$UVWG4HjDW!@?+At$62X`Qu-EudZO$hP=)X^%O< z{8u~rZ@H@Th*i2Wt^A+CD&54>6JTZkyH)yj6{-FZs|-vOs6p;Gv{;~q2{&@Ao&I00 za`Aj_+H%i)f$}EnlyAS4{;yTWcUmc+dX*iZS6XDwjE(+Jo9H5K!Z8<`{4g8!vrY72 z4(9)C@+fYPg8}!F5Sx?++n8^)p-*j-iU^xjhTE9`Y?B`mre=B){U4ju5!QIlA@#_H zqYi0Ynv|d5-{jD#X57rXZ0M8Qq=k4|<7{F@+9C>-J;jDDiC(oQ*m!@v>fnBzE{AmO z(kUm;*PS>iJwtlco0Y5jl5EnSX+!_VhW?*k4NdCRFmiM&v(t{QWC!z#tWm`B4e`CyI}4T3SE%Zc`B`?! z<9@nb+%woNIX=5+gY#7OQU`sh*v9y;O=>3Fq;{6L&b6`rU!dw8ZPJilq#766SpO+fP19}Y@O6qg zGhbO^+t^R=nzZVQlr=U_*}~ey&U3Xzx3RaoNI5J;ssq`X)y6!eO`N_q>9%!BPhT75 zzm2umPU%MuoNklBr8XHl*CxZGjb+jQUo2J{pGIY8TSV)! zu>NP^yEQ0Xm_g+s^Cv8{C#|B7wMb#Sll{*Y>OYGZ0-REUERD1Nm<)TKG9d2fp}IZKquSE3q`IceR@U9?DcYB!zK7RiUaV6D(w zO2J<8KfM+G|5o}Ftx`Nm{_kl;|N9LoMV1k^e6olBNUK!N_i+B{8&aLoBQ^RQRf}7l zgZ!UwmHOFMX_#zf{MRa`rDE0OYGq!&RGA}M$^TN-!u?u}J?MYrC|hDH``_M>w(3&V z9;sE1jjhr_xX$oabOuY6v#(TjBYSwR-Xsh239ZsU)r09w9x^|}ECp$&M1Pd}YZ?8WxZg8>iT(}~za>oKmL3!8e~V-% zuNUn`=6`2LRc=Cu=pZl5r}B5NXMUgg|EW<`I6?W3^{C>#>%}l_P$lX7|7PEMDa&O2 z-!-Z#cKKB0PLHYzT`$#4^43H!|L+)8b*UX<4B#I<#NQCSp8l>+nRKJ7Dbu5x^`pw1 z?NgS_QPsjBSgqCTDgWJK)2|o1k@_GnycA^UizWB?< zn>4OGF_*Jfa8$VhM%5_uaU-XlGK?IGyqvj(%Vhxip$~eQqw7f-Ro%(V$)#P6ZW42J z$(J)%>l8=Q<cid?V_dh??JH5WLoHGh+>K79R#g$MvQMijGP)T9WUI2+ZuZP|OG$D!dosGEm~chD zZYc~dQTipy`<^!?e;4(Eshc^S7Rg2C7^yEZ-jr;lCbCP8x96&f`|{NB4KAch9lx_g z-NpBIyiUh|GIBZ3K1WSFkfV;9k;TYu8vfUz;oly}T#c%~g`p$w7st1%ryr0EkHG&x zC1ijJ_QGK}ewO>>a-RoA)kK#^X(l-Tk#A8uBa(y6#jaaK|BUZ7A6bC?5x(^!++zxz z$qASN7h~0pl$El5jK$MgCn~sJ;;|>fPS^#>&?4J7WBnE}f51N5kM9us+1sS;v+Kkv zw~Fm63K~j4+4N7esbP|{Crol}UO-@MXJ3Rr!#Q?|)2Hx_qi?_o%9N_*BiV)l$oKb&gIkW`?o;&%8`>7~g-8n9|2o z6ZU45kIarRu|SI}Oj^OJ4-#8Sr`TOV(xw|z?Y=SPFoj9Syi0YKhe_ADFxG#=q`RE{ z|6-W*E(A$mj8FAb=MBK%^r#v#hSC2Cl95=S8Vzno-(pO;iQ5xE|KAqI{7e}8UxH*j zz^60;F8+TpuCy%3msjOQcT0ZeTIvtxzmqwCbE8WY&3ILD+_*9v@v4#p zmnub;p`%fb;#Ng=hg9xfD^*-yjWSFPvUcaVs@uh(hX!Z_6Ey8*?zgW)%*+p4 zOyjDhdR(;*trhFcsIocMihX)5`af%>T~Gi2^jhi2M!zh{tGf7ZozUIqQa$I^O7A)J z)8f6VKi0wcXIu?tua%($`u~QtGNM~6qtnbsM|hQc!KFO&)4~VPC9iC+IiL{cJYj?JM^GT(9QE&lGd@t$=b$#uWCWs_N)^tvJGhmV+LuY^Qs19 zy}_+a`gQ0?FgL1O#~xval!fO#wUu4ts%FTo^u~4QsB$)H z{J1JyR)@4^(*I8` zQnrjD^glbK4ccQI^#4fP5C{9e8UO9<5NA@6>fTqRdQyv2FKtyH^e2)3Fu30#LogiM zAtNzG)Zsg?&sPE7h32Y2eo6eXox zQ%1P*g(#^QijvC7D5+YCV*D2+H8W9CI}s&y*p1lh@oO+fiGzFh!7TUoA*sIEI}*kC zKT297qU0!Uwv=eGCqzkGSd_GfbItx}nZUg>E?T+*qS1efmhQ+X=0Bq7|3}exkCOiQ zC>hutC4-sl|J)NL!>Lg+k{l(Y*j?D&_<16tWYK?bI1T4OW7tanX{%_R?0@xbW&Lj} z>wjBW|Jf?}Gh3x#eyiwDZ{_@_t+GV;0NS}=2!(JcE9c*WPyv-t)wflup=M|+_1`U= zO>>JF_uRty54T7|#4XZjyoK-o7RvTEY2LMs@!vMF9NWhDk9LX;W7dgnV&fot`}{WM zf3`{cP@Fhianf-*PCEPQrE9WAoPD=T_t5PU!To!as_6gL(EqKc|64=w58v4IAa@3#xSsSD!dxOLfpVhfVVvz~ZR=z<} zk&bg)`2WHd?!QI4vbTs6x=kCThiiIW8>DYxgY*w=kiCQ*T-v~T-v-wEHps}t2HA<5 z3wtusW8A=617!^|AQLnRTO@mVgJ=UbNKV8CNykl>vPJR|Hn0w}K?=e*py#uN^xs1N zbqn(-TNr9}q5qmv;4G|lt zTW=Cm@az14;U;MYjdO=&5A9$KwS)05=U-sgxn4s5o%1gyIR9di^DmY;|HAkZb0?fv z5b!PT|1Bvoyd zjY!k#>=v$Fc_M4G0z=jKMhge{Pr^ zkKR?{*LqhM^spT0Ww9obzi*23Ptbp0{k;&1SdTAWIwpop^dC~tf1tf9OFKqil=c5a z+9>3O_5Y)+|EK;x^nW>bq$ve`OfWmpH-#1#`hQ@>YyAfW9eoETR9ijQ$Tg zK2CIfx{*EUzmr~c0Q=xS|IfxeH1VDcqw6zbL;vsCd#p#J|C5IP4|p@t|A8^gMZ(>Iyc6z&FR?#_ zd=$P(U)Dj`Utng*Lz4?ZA5xDXLy(QgIxs^o41$yJFCve_2{;LTxUC`l5c2QX{{jDm zDcYYaaDOwN@gnzf;r}h{p}6nB{yp&F_Cw_VK*z?<`0p?0e%LtoW8!`3-gy?iO3qo? zb(T4b&!lDFJJL$oFuOSS1MJnD17hM_5W?G#?vOj=e~D)l>4M*2&WEGi;|TQ6aLz|C z=hP&hkv_`QaMoGoSl*=%#W^);cSt19R!aC*XoC^>Bm55j0NSB{$e)m!Wy&dKaSrqQ z@K0F5IJn-&xiZTWQinA19!)8n0|Jd;f{JX;rAb?0ocgYm_i-)__S)>vSf_Yjs*OA3 za^koFZsa+(Ag_X3;9Ky0h{OF6R_agrbj`3dqjrFZ>uLaLY&DM)(2bB22<4vAqr)-WY+s7+4`RJ%=>1fz^p5G+F`D z2(Y#GAPF(j%;*AfgyrbE@B6;*>Z7{4yY9PcnW|EE&q9rc4y2hLBhO6NnThORW7!Dp z25Wzww_2<>?5Ovb`5gIV<~N_`ec$KF{N-C`*2l%yX`6n}_W%1#Ux)ZR*1sZN{UQD+ zTKBxZL7SGhX!{BNc)I!L`|hv*jK7q>o)+u)>uK4(6}o(8eEtr9{;U2!#g+ZIw$Gpc z=l(y1_{R5)?;G$N`$hlXLOe0IACrIcqcKI(bcuSO@%xC>|HpnGkp?Xf(ePin2bNFk z_foWd+3zcU-R~>@f!|l;pXjo`oA05AY8_KY^*{7u(Ljwq@ng|M%|G&I(efid=1bDf0pm2=Ih3X z`E07h{l?%2{@m|ev>E5Bx);ONN30M2C7+MD{YU&R2JcR*xXm%$;-0_tI~n3G>sR$H z%)Fn^M||ZmzMfC%Q?PzbJo^d0pig}xCVqfV$i6YhOk2LR|4K~q{7ljM6MjRTR%wm4 zUv=K7G|y2yFW&{_+!W8rWBDiaY0%<7)TeRwLL}G0zoa+m-_eh{&-(}TQ}omHhpv$y z7XK2xhra8Y`gQUDYWw5juTYugYWkYxdMc;Ck*--hO#L)XyRN-uaf7rkh`Yq!68~%J zxBSEU5`L6MEKkzUS$-G&t-O9o{5|ng;@_sa}eHHS%ON zQL{E|`TC2|`s8=?e>>0A@!da-&aXSKpLL#p@}Eb~Gp>u@;=_3Oh3Nl;>*VKM7tXx4#Mc3x?mCTQ|+oL8ErOLX~V*9Bdr8M^kc>w>P+4Z8Vj&MVE+0xf>od8K7q zq19h=UC{dX{`c5Whs`I=YgdPDb=WE5?|Rnt+_)*;@|?MThfj`H_y0w#iI;6(v42s# zWdD28w>}{r{T|=n z&|k%{c!UONi27-Oda3x|38i@y;?6JsRdk8l9or$VR&krWn#Cw^yBoO{+xSzdd>Eq zw*C{AU$=ZjiMnn3hqjH2r|JK)?O)Nh<)5_ON7lu^u>39hN7nzH^?zykXD$Dr&-{O5 z`G>{-iSOi}692sIZxcUGAE1Y=|G0hsB|T>OzZU;9+kaC0dGXiir!D_uYPbA$aliPV zB=_yo&s$!mcUxW;|L5)t9+T!>^nPhROy9BnU+ZJO_~-n;f64#*SNea&mDa2NmjCHm31|Ib7Ee;(5R^N{`@|D&Bop8h{#bnBrQ+keRY??e0$59$ATD5k{I)-Q=K|J_4z zMSRuzjQE=Uv*PR4Z-{R?e(wGLzx|UB`JD#;-~R9J#L{m)p6`BK{152AqMxE))@DB~ei!``{T#iAo}$O; z!}L-5b$Xw)AEgf!()M4m?Q8VE(eKz+F8&SillJ`@{ifwl(I+gA+jo_&(>g8F3$#t& zpcm;c=(nW3C;kq7%JTmy{+vAio%jcx&!=tsQE`{g#b<5*GJV#zXXstFf6aQkxSOui z#U(_UEukA+rr@j{*^68x7 z|DWam7x#Du=>4quR?nGlB_8;?`G4Xe>%-!aCwBG!x&NnenxILVqG`HBm)~&zPk%rE z&+;r?ryKPDd;dRXJ{qmi>eGB#v`!neNn5l%wHrHQyRlxhfj0SzGwp2o1v;Q_TfZl+ zvagSv=Pq@#w{&maq;tSSS#2*%y+4kGw-=*j2YgA?1rv1~F=c&$m6D?c*Q`_4te?$B) z=$q7K{a@SnZ|Dd8&cF}TJ=Ro>{=I)2y{?Hq`}(CR z{r~nm|2B%>Z+gG{T70)&bdOO+IdqivcG}um z+S*y#+F9Cq$=1uX5N|2g|6D1r%H!3>La(+7n+lbdnk_Y#mRc;el$Kg8wU(BOv~926 z6t>4(?w9u!L4W*Y{G`_}$4|w76aQ`eqxk9g?-=HO=HZ{Ew--Ta=|ZveND(|*1dkQL z<3;d}B6w#JJRvBOT_}z!k`^h8ghjd{S&^zp^lo|&y_ft>;Lp6DK0v=hAEXb_hv`@8 zBlIMFl>RAwjDC%No&0d{&wQLdK~K>q={M=O=(p)<`W^Z-eTF_uze}H^-=k;f^YjIJ zmY$>M>5KFw`Z9fmzDmDOe?VWOKcqjRKc+vSH!3fd!tzpBQ3|U{VRb32DTQ^Vu%Q$- zmcr&z*is5xOJQ3nY!@D`7I_}77P%d+7RMg0>FoWP!}a3W!}ZQ0+?XiB&9YKh9D7tX zQwnE`uzs-!n~J?hO-1^n)}|tC+bqI?fg&9A4|RU#Xm(n7+%r{#gSDlwDDimkauJSR zEy9~cZYO0${wH;7ML1BDelodTgi}S`PNs@-PiuMHw&&@ z-dltXlSMdG)F)pm%FQ>6dj56!wIb}QEW+N&zaD+C@ZEvZ!f?@6e?3tg_18Cx8vgZK zQMd0_6ghvlv1p&~cFF%|+?~_LK6p zcsL$@Ydrk6c=+vJMWiK)aIjWbN+3nC|KZY(hf6yiF73Dw4_~n3LTSf^(vAzI9goDr zkJ#}@X~!d_O^?RIkJ|KTY15;nO^?OHkJs-bdq+$KsL4-weP=xSL_Au`phWa&acGI^v3TsQ z;k@dCf6V^JOW`}C1o~K!s&u9wkH_B{PIU?U@nUo7Y`-Jk@z!v<-%+YmsVUx3>@1!4 zcg8#48cuwvVcuD6E}i=);)%D0lV574CyLFb^Z%}R*Smf({`>eJ;uphDK>s5BQv7oK z@=wMu|8o43cz3+}6Y=ghYrCF{_r!a?9Pf?yzAxVU#ka+KZTrIK;(hVH56Am{JKpyv z@xJ{n+K%_f2jT}K6_?7p>uYB-N;#WTQbo|O6#|PtsABzt@6(9Uu*!ZFNQ1PM< z$A>-ar79aayeC%US(o^xVUyqM{{QV*Iuf?y& zuYWauBYxw3@f%OZZ&>(Rd^|q>{*T1RW%%(Y;}h{zJoO>3k_092PsS(T8=w5`_~h@z zZ^mza$m>J#Tk+fRKgRzQpNfAL|8x8=@pL@>w&&g!PyfQF>8tVdi`(%#@jFj{B|aUW z{)I2nAH=7hel|XB^)vD5=e#b^Bk`H|%zNT9PseATjn6#ib%AXCllY95{A_&osrc+y zJ{P|mzgxVXp=YCrgwMt2;&Zb6z4$$BzxQ-J6VDVc`g}a|d_42Dc;=7d^YMlF!n@-O z_`V?77oLhQd@{aJRK>~rg77)73-m~Q;rHVUTJqU=_Ve-VbMfr+@$45r7tj7tJgf1a z3#lY}?%na+Q}Ns<e#J`-P+i#i{1LzZHV2**lmj4=GbkC z-L}|mkKK;g?Tp>7*zJzpzS!-L-GSI0jNPHw9gW?w*d34EiP)Wt-Kp5U6uXyW_e$(u zjoq2py%xK(v3ossZ^Z7+*qw{r`Pf~E-No2lirwYdU5nlI*xiWT&Dh|KeytFbo|d)H!b zHukQ^-i_G18GCcFw-|d%v9}z1E3vm0d+V{c5qq1lw-tNav0oAUm9bwH`_-{u6Z^HX z-w^wavELN?&9UDS`>nCx7W*Bs-x>Q|v2TE|C-!?|zc2RtV}Bs_2V;LI_J?DCB=$#R ze=PRLV}By{Cu4sq_Akf&mDs-;`!lhBE%s+)|9b4-i2a+fKNtJ+vA+=ei?P2H`^&+h zzrPy$Yq7r`ocQ~j;SKo1Rjn??aXnbQVzA+QuxDnsD8Mm6^R&(5HiCe94t1WJ| z$E}XI)fuWN#uajP$G^~bG&xHTBJhU3;q+!~ErV{vOdZcW6k>9}<%Ze5OB zSK`*yxHS{EuEnj{xOF{l-H2O?Sd3dsacebht;MbNxU~_tHsjV-+}e&?J8}DB+%AjT z<#D?rZdbSpw`<~dZQQPl+x2m~A#OLu?WVZh9JgEIc5B>ji`(sSyCZIQ#_g`S z-5s}k;&yM`?u*;~aeE+c56A71xIG@Xr{ngexP3WpUy0jSS2#O;;1(;9c$;!b5MyFai=ft48)zmxHA%W#^TOI+?k3y)4}|I=StkU z8h2*m&TQPd5qECJow>L(A9ohw&SKnIiaV=uXDv*Hc(E*AERPo};>Egnu|8gGh!-2< z#nyPSEnaMo7rWxcp?Gm5UR;Y8H=hrWH=b=g8VNe$pgIm}f@j>4w~bjB@SBS zpe+vC<3NJWIOvLl?l|a)gWfpki-Z0+7>I+xI2ek9;W!wHgV8t`i-UUc&R2{s*RWG z;-&g{sUco!jF+0?rRI2PFkTvpZ(fXVmc=(K;+r+`&BpjosGNKU+!lPh)$3a;MX*`~n}Tl-#<$zN zij}r+?Zmg+y^56%uOjaBD&nqhtrdZ#?jqc)T(Z zuS~`(Q}N2Q*DIHb;93#P7Qyu*m@9&L!K+n8P+bHyMNnG=bwyAwc&*s+TCwA`V#jN( zMbH&RmX}_wc;#}D&!tx@{Yi7lBfVfTUb*a5lz6!)QBdS{xhPn0zU;WmMfsPD@-G+V zUoNt^Toiq|NOC1!x$0HqA}CT7k-bG^|J5RE!Fht&^8`w`s)SPF>*t9}rPZQDv_-B(RHQgx zwuIt(QS|lVs8TbO8llt#H{z9>UV2W7+RweR@M`U=l@>}HUTZ9>6a~_PSCNOH$VF6S zzi_@>s>wo8j^KQIDf5LQ&xN9Dg3{6FP}a^jf?X)3FDmQ;Su7F>N~=ZQiv`VjRHP`1 zSS&JKENwkMLzd6ilnpaVrptO2kS?lL(rOH_@jw>QZuAMJiTK6iB zT`$rKz8^_jtQ9GWwCk2$z39CwXf6e>wiQuJDL7wlEtZQ8I8iBRk5|jQWG|R4g6pL~ z8LyT-2#S2mlpwHF?p35Lf3;HUilh{rZz}3mtd$ZLhm;pf6<)7gDgwn+6e%l;Vk%s! z0!IkG9~H@q11pLH?XUDXD3~aMsUnb1Ws#5IK~x-St#m+LRV6C{|P)QbbY|kt3_m_ms-Be7>i|SzVM?{mNVs z%nM4#%D6c8LH<7HEaR;{-zq;VHAR^A$r^7n5rpFE3EWKmb_ zkf6ReOmM!e?)B$wUtiRzzNk}uQK$MMTfz6E;>e;--n~U2!}?MVMeYvNl8#)9SJ%U& z*H^dW)t$I^G455wy~?;(9rtSDUTxfKjC)OSuQ~3u#l7CRHx~CM2#>+!X^c&#B`YYg9#uJG47;`Pa{ z*C#hc-RDuyc{F$)jh#p1=g}3(X3iIHoJVuEEXM28W3NxkV|x5Nx>1TQHN@+e8sqiL z-LGG^^K#F5Go=^=*|%_! zYN%MYZ^6DL@lp{>v(it4G(^Mk`dZWLtMl>t`poO=edp0wyuQ)&`nrvqj@i2S`u38j z{Ppb>QR8{kb{=(~M+4{4-0NEo+7f{mZL7+5Yw7Uq_EM7V!SiVBJeod_X3nGezdvUF z{LqEJ->^_@c;n)9yiqY2`n#!&%Bg}XsfwzphH9yf>ZySmsfn7Yg<7eN+NqPesGo|w zhiI5aX^h5cf-cb&nxShnOV{ZJ&C_zcQKep0%CAyhRS)%2AIYmqURCm{l2_FvO%a|d z`Bq&fc~@N}`B%xmO8!;yuabY&O`0R+R4vdVEyWwvxT|qj!Q3CF)vBGtj?>_u1dQq-KzAfSXPVe z%2-vWRpqQYhpWn3bq-gRx9S|ODs$C2Ty+jtF|R6rRh?IHuR4#b%3M|Es7Yxvjj zui;8v@UP)t!@q`q4gVVcHT-M%*YL05U&FtKe+~Z{{x$q-_}B2S z;a|hQhJOwJ8vZr>YxqqLe`5{*8vZr>Yxvjjui;8v@UP)t!@q`q z4gVVcHT-M%*YNuhp*Pm>ui;aUdZsPQ{^2R34O`MxJH*s#_ z+{C$wa}(z#&P|+~_%`uv;@iZxiEk6%Caz6fo47V{ZQ|O*wTWvJ*Cwt_T${Kyac$z- z#I=cQ6W1oLO~J@NVJV!n=ic z3-1=*Eu338w{ULZ+`_qqa|_=VzAb!P__pwE;WMG(jVMVr7QQWfTllu{ZQAkwsCE{CbzL|W822Ijcps-HnweS+t{|TZDZTUwvBBY+cvgsY}?qj zv2A19cJ8+EZ98|{7`HLHL3v{v=eF~3n7B&5KL+7Ex)b{7Sr8?Qmc7Z~B^lvsyOQ{WL(= zWt}YR zWLY;#<1|ZB*4?0+G)If1@^!1U7H`&HB*)h~zP^fTNgnmr$no`#uXlWdJR8PHz6}#} zmDI68x(4YwVRcK>(-3b?%V)Ye-kcfwGo>tz#r=zAR8F;2NA=V|jnqWV)IzP)M(xyD z=r`{Bd2}(G!}pig8y_qkH4J@!>H7Eg9@lj|*mm62^Zlj4YY&zTSATzL;KAM^_ipNm z`(Qk&ptL@9xw0yeI{qid|T2k-w%Z2W1lkyv-!kiiFiY-^3n!2v< zmtQSZ$`VSkq(lu+UX(ghDx*l@02x=DSJ=wz{fZ)~J=Y4AHWbBLQl$cQw$S~GslQKX z<-xXD)i32&lv8ZJUoloHxAq+Ne3LpqDDS#t*UwA3e!t)^%C)2(7tdF2*dV?7g3~NT zLDpr2rpZdtT>rkz2TN$q8_S6;pDWU`JKQgyr-DT*pGT#$1WS4KrR_zoLzWh4sZdd9 zsRE^y(pf1LT>!dL%3UetE2WlraOzfVD9Uo_ly4L&wOes2&zEW~6_mxbv`ssKwb1u# zS)BHYa;l(8s-kMDDO4)4sP+TG3a7rJuGG2(LPh=kie^y*HIf!8tvBC)(Bu^@wzQO* zvdFu&PzlKSow4FQe0~A{`|3Wppo`?*TNNFJ&J%Z9?7Z(XcO@3}DKfBLoUijGtz6-K zT6zn7|2iuw>@6`-DzyOVD(xwfb{D#@rkb?;(s?dDws{*@^cMM+@05-za__bKynVb4 zEBZ>USG1)!Wbr1`yrSZgD(Y;>>2X?$B^QH}R54Jfbd?m5^Iqh7zEsp@uu!RtqK-p_ zO0DKl2_D&Cl-=W!pBR+1WgtyffcnpUn(;4iBjPcrNSqQ z&!?gam*_HGp`vKzs`i8PX{Fd+9I3rKWq7`%!JMjMb5WgZ=M@wc?c<$xo^#P01+@;TB%jODT~WCiE#dk4{uWaX3&DhtrO^G#!UmuG4ZHUY(D_YYTCBqd5*23eD1L z94;x}v;CoGd+tH1qk2-7=l4U;G2JOU=dxfKmGjZgb^T_w(k&FMxTk^;?c2s$lq|u*qR8>Vy)Jc;xMbhZA zInrmt@kq0@5=Yh2SIf6rdN*lD)m_w0J*13k$5%_QPv%J9%TcvsYU-$w25Fc^$-Y{7 z)mBgqIlgv?lvk_1wHt9%S53;Pne3+Z2IV)%yJ3yw z(}=B6K8@R7uh`$)~A}v_q40P3qgEK20}Cc}?EbA^n7NHL3hbWlw6CNp+rdyw8H8Np+fZZB1S}o`o@~d8eREL7!61DNR2m$&~Cb z$^Md}F01Zkhg`A!%5)rEbsBtx99>hDYuj-&t9fQ&%-VNdRj)gh*R9`hYHrBmhGTE2 z-VJ%)(A+nizMHCcQz*5_P8b57r!W9Br$yr$6OeB>X79WBXoNqI|< zm+W8KjH6}9%jbB##xxuc*(8<5!%<74`ISaOACjq|50@m(tPN!f~yv^eP?c zK|0cVbfnklNUzP29+@LOF-Lk}j`W-y={Y&lJ94CVRirMhm|Dlf5O<-gU=a-E@ssEyGL2xamPIqG!@6a&Bybq z(W|<>cJyI>#L_E+cQ~jALK=$8(qBcpm8dP8=_2pap3cG~a@*TI%W~% zqV$XEzohO-KBrhCQX}qx%^Ms=HjGXecSD8?~RiVj2-rMOvg#5V>)f?9HB{4?@sKU z^6Xq7ZP2OSo!Y5OeY=#`Wq-G_yW8W$+u@`aOYacLqpzHl*@w4J+x2Uoe#`^P^rd9F z16?B9y%A0Z%ShdQL7fcY9g@zM)XA_kBhrt^-vxg%rhUh><#;2t#mR(p6P72`ZSpE< z8zY^?^?%}<=41-%lzRDv71Jt68B^*$r7qLToNgj{PPdS{Os|q-FUi-p(#a)dUDifd z)aiqJ?FL8 zylo5O1@&6c77IAN4^G^4o-8_cQJohtFFOBA+Hy%7EJ?R4pJm6cIIk<}VZiCcP3eh2 zniDsfCk9&`sK=oj5J8qXwF#`8YLTd0J^-O)pK*Wx7iC*T|=4Gfr#euLti` z&)sRQblw-IwIgw=tL;=*+o_wrQ{8K)uKm+`dDpkm5Y3T1>bK&wVTz{Xw9)oP~HId(+*{K*w!Kc4t4D8j?=Em zIPGa7ZQ^F?v{(MU^6kalXM0~O$+J(MuI~e%fz&U_4F-n+gAr?L%0GJ= zUQ)M9j=wCwtMa<4&9ADbYw>i(z8Ps|)O|)?GmgC`pKIzgtNqhwOlI6>IdD_7B)UXnhdZP%WvKFZVNF?q@^N4q+Zq zhR?jSkx^2<&%Lveoj7w1osCwJ{iEtVD(&b-oVi&)^WAgKTFT zj(1yoW?=2iz}cDa=QFpqXPfG3KDo%9+4e9Yj>@QxARxO`4;5TA)Q*qGiJ92`1fK zqjlP#P1>Su+KJ@+q&fU^_~-D?;h)3reV*pDl`+=nOk;Yp`gTEM@6e*wRHljO5LE#P0k?>;3h;9tPM zfPVr10{#X33-}lCFW_Imzkq)Mzx$fBfZu0DTEOofC%MN-3-}lCFW_Imud^*J;9tPM zfPVr10{#X33-}lCFW_Im@BSz);9tPMfPVqM&!Mz{e*wRHr?iNlVJ9u(U&OzNe-Zy8 ze&1bb5&t6oMf{8S7x6FRU&QaeD=p$*#J`At5&t57&nJobCoSUFJCGLf`wmRZK#3VB zd2UII_!sdn;&<B^rA7RU_!sf(Bu$=$ z(jxvv{EPS(@h{?E#J`At5&t57&r4|${}O)BP02lD(u0wf@Gs$C!oP%n3I7s)9=gOs zmw4#X68P(zJx%zgSO8_?Pf6;WvPp zmhih@P3{rW682j_?Pi7<6p+VjDH#b zGX7=!%lMb^FXLavzl>jJby~*1jDH#bGX7=!%lQ49`Lv9G8Nd7Uw2Xfl|1$n%{LA>4 z@h{_F#=nez8UHf=W&F$dm+>#-U&glsUW#?lJ@75sY0k{+_Of`0}73VzS- zX$AiZ{uTTy_}w=q_l-$UT3W%sg5NVvTEV}9e+B;ve$V{L{YF~Bzk*+{Thi;6R`Bbr zPde+9=c1&;J~84YM!dv`ml*L9BVO{nlyu@Jo%o3nFX_lnjChF=FEQdJM!dv`ml*NV zDt<=1#E6#|@e(6m(&?Y{IV48Bw2Ge*FS++gtN0o55+hz>#7m5Li4iX`;w6Iwi4ia9 z8BC0Li4iX`;w47BWZ)n%;w60;i4iX`;w1wKi4iX`;w47BWKbb_#!Chkl7WT9h?f}g z5+h#H)0i0Xl7WWAh?hJECPuu(h?f}gl4ruih?f}g5+hz>#7m5Li4iX`;-xkGjChF= zFEQdJM!dv`ml*L9BVJ;}ON@Al5ic>~B?B3W5ic>~B}TktP$MzoB}Tl&h?f}g5+hz> z#7m5Li4iX`;w47B#E6&H@H65iM!dv`ml*MqzMI5|ml*L9BVN*Lni%ntF4N?mGBM&M zM!dv`ml*MqZq&qxmvp5jM!dv`m)v(I{XdBjFBxP>jChF=FBxn}23ry%URuY`h?f}g z5+hzR0FxN;5+hz>#7m5Li4iX`;w47BWN;=i;w47B!vK`$}rCHLn^7j)7EojkiG2ED|fml*UCgI>}t zof!0zKDET4ml*UCgI;3LOALC6K`(9KXV6Oquo8n_V$e$rdT9ecgI;3LOALBR_jY2? zOALC6K`$}rB?i63pqCi*lCJN>pqCi*5`$i1&`X{j(gyww{2TZ^M5Gy0`1{Q9kuVT{D^m$vXT{iQAZjDJagR??r9 zw(v6mrY-zi__y#IAWh7Gi5W04114s`q~9wUG)>Hai5W04119}o$q-30M3OvLCeM|L z889&eCT76I44Ag@GXp05Wr-Ou8A?gafQcC}=|4;QFOp}`#0;49VQ!o)zBcJS}u-@(sJm<$;uX2Qfwn3xIE4*nhdJNS3-@8D-J zObmvJ!7%OMXEIDohKb2AF&QQ%!^C8mm<*GC_rz$JcJS}u-@(6we+NJFVcNmZe3%SV zB?iRAfS4E%lOe05-!kpsXBtR5`1RkXot?-RFH#wmQw3F06;)FW)lwbRQv)?p6E#x{ zwNe|kQwMcY7j;t)^->@8(*O<95Dn7^jnWv6(*#Y@6iw45x=dH-D$UR}nx*S>gKp9s z&C>!c(h@Dx3a!!_t#Zv-IUM{A_)>3_ojMF2m2>m&@?8_~kPEY<{^6zdqw!hM(Op zm*FqN&-Rzg@U#BqGW`1Wav6T(E4d7R8U8Z-`kJ%8=FAqD_4{Skz|0z$4NGSBz-$~Q zm*Hm-%#04X48Oj?Y-}d$pU(QHb2)xPn7JH(IsS6|<@n3-m*X$TuMayj6K4I`xg38v z{&M{K7;`y({fxOBe>r~rjkz2@BT6pEUyk26P-Z~P<@n3-m*Zzf%;osY@t5N-$6t=0 z$1pP!=5qYy_{;Hg31%+A%n_K&@t5N-$6t=0uP&G4*SDUT`P?#3TV}e-JZ+h$Emz>@WXl!!EAVr&WlpwSf!~mBHg=VH*D}*u=2puU_$%=1 zQ_U6lIny#TT4vwMEN8g_KPy@05z7_$4GCvsUzxcivvOsAt<0sB^~YwOs?0By4H;*4 zsLZ&N*`YG)Q)Ye2%qW@RDKj=@hNsNMl)0EP_g1dJ&$5&mm9l=_%&3$Zm2w6C3j7uL z8J04`Qf64n3`?0|DKji(eZZM%DOcb(nJSx1l})C~#wN1KRJjtr$yC{7s%$7bn@p7} z@taJQO{U5wQ)QE>vdL80WU8z`Ih#zCEAg99l})J1mG~?1SK_b4Z-Q0U=bTNl$|hN5 zldN(jeiN;-alu@P-$biiiN6wmC4Q5xawYys{7gC7w)vdLk&3O`d)uEJl1pY0%5;jhA9g}(}a6@I3rT#cV`DOck+sVtjRmaFks z<2Si1SL3h7UyZ*SKZ8?dVaV0^*%&fcdS-aaTh{Hzn%#I|f= zTV|oiCbnhc7@4^$SL0`}%GLO*@mJ%o#$S!U8hWN!Ox+&UX4$;Pg;v65Va-{iSmgTDrU4So~pvT?p_GF`60UxS~~ zE1OuC**&t!b(!HS*Wj#>_038P2kC$6SlQ7C-Y@ zW<$wl5#(C@wfK!m=34x<_-pYSx6fu5WU~u$E&f{kwfJlCGp=Rhm$?>yEq>#exfVY& zTdu`#Of%Qwuf<=BzZSn43b_`)SqhnXG4lgt#<$E9ka+@fE&f{kEHSwje=Yu6{I&RN z@f)+sd;^(pAlKqIewAzSGtOl`f?SKg7Jn^%o`PJ5-}HuDhrbTLX%4v#KZ9N7H^_DP zO?k+5_*rjq9e&2UT!+67KR-gQ!(WG=EhpFEuft!5zYc#Letv~qho5I5*WqW~$#wYa z@Ymt5!_Ui*>+sj%uft!5zYc#L{yO}1`0Mc3;WzCf8w<|Hf^!{y(=M_x;arE`^owlz zMK)%a83VI%;%uBa*Ws_jZyYbz;jhDAho5&Mo1T&D@z>)wO(UD8k?Zl-<2PL+GZ*H1 z{KlGdJ^p(9_4w=Y*W)+-oa^z|+#p)ugA~hkxd)Prj2A?k6e%6xOc9{Z|plWKIVG-_4w=Y z*W<6pUyr{Ye?5NV9Og0@RH{fr;-+;dXe*^vo{0;b-JaYs72K){989p<^XKujH z7n2+Cvxnsd{EhgTgL5PPM*PNhGh=9G49)CgxedZ^Yk-zY%{U{zm+b_?et@BYu8|+=!o{H8E~&e>47O{JdPb z8GkeWX8a7uxfy>m{$~8m_?z)Jpv&`l;{B8K# z@VDV_!{3I#4SyT{HvGJWxeb3C{xXB;KcjAL$KQ^>9e+FicKq%5+wr&KZ^z$`za2l{WNydLLY&+2 zx8rZe&&Z$K@wek|$IsNC+wr&KZ^z$`za4)&{&xKB_}lTf<8Q~`j=vp$JN|b3?fBdA zx8rZe-;Tc>e>;8?b8F8p2iyYP46@50}OzYBjC{x1Ao_`C3T;qSuVg})1b7yd5%UHDBz$*l3Y z3x5}W{@>h%zYBjCev_?o7yd5%UHDDB%3b)o@ORF8p2iyYTbveVa{N4Dw@pt3z#@~&f$1r!}@5XPMLGH%ijo&1n z+>O5*zsWwC4>5P+Hw__oF!QX?w2R}b} zHa#Nu;P1iTgTDuV5B?tfJ@|X@_u%ir--Evge-Hj1{5|-4@b}>F!QX?w2Y(O#9{fG{ zd+_(*@4?@LzXyLW{$BjO_l@$<0fUi`ej z*?f-Ni@z6tFaBQqz4&|a_u}ux&lj3`-g7U0zV~eYNAAVni@z6tFaBQqz4&|a_u}ux z-;2K&e=q)C{Jf>P7k@AQUi`iId-3<;H*qod;_t=Zi@z6tFaBQqz4&|a_u}VM&VBg% z@blm2KKy<7`|$VS@5A4R-^7&Mho6@<_u=Pf&3*X$@bfrllgM%({yzMD`1|np;qSxW zhrbVhAO1f4{Lr}%e;@un{C)WQ@b}^G!{3L$4}TwizUgdoTIQY3efasOb07Xb{C)WQ z@b}^G!{3L$55EaCxevd21i24C&u#9*--o{sKmTnuzaaPF@5A4ZzaM`;{(k)Z`1|qm z=;nU>{Mxx6e?R_y{Qda*@%Q8J$KQ|NWS!iPzaM`;{(k)Z`1|qq=#hJG__v7!!-;ci^zlp8c{E9q)e*pgg{sH_0_y_Qtbd(4158xlbKY)J# z{{a30`~&z0@DJc0z(0V00RI5~0sI5_2k;NzAHY9=e*pgg{sH_0_y_P0;2*%x_ni5j z^8o$<`~&#S_sD$Ec>wC8#od@unFqH@J z58xlbKY)J#{{a30{3cUnb4l_b{z3eM_y_S1;vd9sqE$B0Di7ix#6O6C5I^sJ9>hP0 ze-Qs5{z3eM_y_TuK$%U>%7gd^@ekrRQ7aGPAH;9&N;Y>T58^j@D-Yu5^Uj0#2k{T$ z=T*+;2joHggZKyW58@xhKZt)2zsZ?-5I=u-9>hP0e-Qs5{z3eM_y_S1;vd96h<_0O zApSx8L->dA58)reKZJh>{}BEm{5%fNA^b!5hwz(dmWS{U;pa)uL-FMA%sKZJh>{}BEm{6qM8-188A^Ca>R z{vrHB`1!c=5dIdA58)reKZJh>{}6t2EAkM2li~ck6T&})e+d6DeiPL5F#ciu z!}v{@%jRO_Vf@4Rhw%^NH@_$k;~&O9jDHyaF#ciu!}y2s591%kKaAg`yF84482>Q- zVf@4Rhw%^NAI3k7e;EHT{$c#X_=oWi<2RQn591%kKa77E|1kbx{KNQ%@eku4#y^aI z82>Q-Vf@4Rhw%^NAI5K9Ngl>OjDHyaF#ciuBlt(~kKiA{KZ1V*{|NpO{3G~B@Q>g( z&mfQBAHi=ja2~-wf`0`62>ucLBlt(~kKiA{KZ1V*{|J6_6!HlE5&R?gNAR1okj;0= z=4<5<{3G~B@Q>gh!9Rk31pf&B5&R?gNAQo}AHhF@e+2&se)Ar(3Cei{{|NpO{3G~B z@Q>gh!9Rk31pf&B5&Wb0&6CKZ_($=N;x}g^kK!N2KZ<`8|0sU*DDo(N6QJ`b{!#p+ z_($=N;vdC7ihmUUD1H+yvw0VJ6#ppxQT(I$NAZv1H@_#3;vdC7ihmUUDE?9WqxeVh zkK#9PBah-A#XpLF6#poGKdO*N@tcgDO~%fn_($=N;vdC7ihmUUDE?9W=6>W+{G<3s z@sHv+zbKF4AHzR}e+>T^{xSSx_{Z>%;UB|4hTmMIJci%Ax;%z|4F4E@b4T+S{xSSx z_{Z>%;UB|4hTkOdJcfS^{}_IAQ1Tdl^P2J){xSUKH{~(>WBAALkKrG~KZbt{{}}!; z{A2jX@Q>ji!#{?94F4GZG5llr$MBEgAHzR}e+>T^{xSSx_{Z>%;UC9u&TAgWKaPJK zzxl6u9RE1}as1=>$MKKjH;*un<2OGxkK-T5KaPJK|2Y0}{Nwn?@sHyl$3KpL9RE1} zas1=>$MKKjAICqAe;off{&D={_{Z^&<2O$ykK-T5KaPJK|2Y0}{Nwn?@sHyl$8X|& z9>+h9e;off{&D={_{Z^&;~&RAj(;5gIQ|Lz6Zp+5%MN|@K4~Mz;C`$p1?nWe*(X`M|lFjIc(V+ zq&$It0{;a53H%fIC-6_;pTIwXe**sm{t5i%EM-3|kSFl_ae+L6e**sm{t5hkY#^H( zmnZO>AD7Mf%;q@d3H%fIC-6_=pTzG+3GyWVN&J)e{XjvU#6O9D68|LrN&J)eC-G0> zpTuv@U7o~0iGLEm`O?{s8stg*llUj`PvZ9@2YC{|A3DgB_$TpC;-ADniQkVOQK2|nYHBaK7#6O9D68|LrN&J)eC-G0>pTs|je-i&B{z?3k_|0?8 zllaYb&6D^i@lWEP!f(!CHpe?p;h(}kg?|eF6#gmvQ~0OwPvM`!KZSn^{}g_6WwT#r z$W!>I@cWg9JcZx<**t}R3jY-TDg0CTr|_Fso2T$k;h(}kh2Jkb^A!Fm z{8RX+@K52N!as$73jY-TDg0CTr|?hV_sbA@3jY-TDg0CTr|?hVpTa+de+vH;e)A)< zIlXxrzj?iR8viu@Y5aa(Et@-;r}0nYpT<9p-(1Q(jei=yIl$Q*;5?0g8viu@Y5ddp zr}3LVm#6Vh|36jTOKxS2x@Kjv7Zhm&P0SQo@TC9@@$=ifVgw;?Hk5DnrRujD51 z?1Y4*;2B;aYel@F2zz ze(l};vHy?#f9(HbzZUQQ*sndlKlcBz|BwBD?Eho`AN&8<|HuA6_W!Z}kNtn_|6@Oq z`Tp3iS-wB^zu5m`zrOEY?0>QU#r_xjU+jOe|HXcN^S#*rV*iW%FZRFK|6>1({rb{- zvH!*X7yDoAf3g3?{uldS?0>QU#r_xjU+jOe|Hb|n`(NzWr{0VGFZRFK|6>1({V(>v z*#Bbxi~TS5zu5m`|BL;)#Cx&-#r_xjU+jOe|Hb|n`(NyTvH!*X7yDoA*D>CU{d(AY zwV!KnulB#%|7!oM{jc`F+W%_*tNpL`zuNz5|EvA4_P^TyYX7VKT!eeI|JD9i`(N#U zwg1)rSNmV>*M{G#{jc`F+W%_*tNpL`zuNz5|EvA4_LEKS)&5ueU+sUj|JD9i`=Lqq zYX7VKulB#%|7!oM{jc`F+W%_*tNpL`zuNz5KXm(E?SHlZ)qb7&z1ja}|C{}9_P^Qx zX8)W0Z}z|0|7QQ2{crZa+5cw$oBeP0zuEt0|C{}9_P^QxX8)W0Z}z|0|7QQ2{crZa z+5cw$oBeP0zuEt0|C{}9_P^OrbGbMB-|T<0UvqkI_P^QxX8)W0Z}z|0|7QQ2{crYb z?eES0H~Zi0f3yG1e*OKu+5cw$oBeP0zuEt8|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_ z+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~ z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g z{qOd_+y8F=yZ!I>zuW&||A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F z|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIP z|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIPuV+8b!hP8PVgHByANFh9 z@5BC2`?c=(Y5%AFER*}R|I_|Y`)NY=X+QM)KJ5p8->3cX@B6g>)BaEUHPH8I|EK+* z_J7*{Y5%AFK(qU_|I_|Y`#&x%U z{xAE#?EkX=%lzwBqv+?W0QnftQ;%lzwH0A|I2>1!F}8RZU49Z-}Y0Y?%V!v z`@ik~wx54+-}Zmo|84)b{oIcGw*TAyZ~MRP_kR-HxBcJtlcw(5{%`xg?fZANzmo|FK{HfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{@;J0>&<@s|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs< zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a zpZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmv zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>! z|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^ z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y z{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ< z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs< zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a zpZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmv zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>! z|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^ z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y z{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ< z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnx*Y@lG z)BmUcPye6(zkhAN{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(zkg%D{y+VH`v3I*>HqsT_Ur%C|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|9i0i!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT z!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?AQON|4;v){y+VH`v3I*>HmAM z|H1wT`ycGr|EK@&!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT!TtyP z_5bPr)BmUcPye6(KmC9D{~qmswExlmNBi~v>HmAQU;p2u{g3uP+W%<(qy3NeKidCj z|D*kn_CMPHX#b=AkM=*>|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*> z|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*>|7ib{{ZIBk+5cq!ll@Qj zKiU6e|C9at|MdUq|9i6k$^Iw%pX}HFr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGqd$#}C{%8B2?SHoa+5Tty zpY4CP|Ji>1fBOHP?SHoa+5Tty_5bPr)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIa z#r}V>U;m%}KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq`^)~n?AQON|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|L^VJe_6r+gaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu1JJwu@Akjj|8749APhk7 z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^VY0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da^kM&p{U7#!*pC4S0}uuv z3_u_Df7t(F|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy9=Sf7<_P|EK+*_J7*{ zY5%AFpZ0&+|7riH{h#)K+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+|7riH{h#)K z+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+j{yh+(5L;M_J7*{X+H)a3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE)om;GP%f7$a0Q6=5m;GP%f7$a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaPpOf2ysc0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaM~?`)L5t0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0jb`1?r%hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR*uS=)1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0Q)!g(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|*iQo(4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|+D`)*4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e7Y zll@QjKiU6eKMi0sfYAU(1K5-OPxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMG zWdD=>Pxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMGWdD=>Pxe3A|78D@{ZIBk z+5cq!ll@QjKiU6e|C9Ys_CMMGWdF1M&-Opt|7`!W{m=G4+y89;v;EKZKimIo|Fiwi z_CMSIZ2ze+fM@+4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y-~ZBuLWlqUABPVI*LfU%9Grb|cyjPM-r;}$$Kn5Q@TJ|s&;1Yo#eeJV@Bet` zzx)5as9@IU>>5B}%>_`$#P;NSkQKJ*_?`G3FT!9Skq|9$X}XZpAQ z+kg0wfB(*Z_>lkbz=!;kt6RVS94i64`o!@^E2>kx#(*5sW&V&B`<%HAkU(V7v_`Uq)e22q--R6=(C=UW&Y$vM?$!PM%S}$df4OVr_b<18{Qk|k>fgUP3HkdsXJvl>=4Ha~ z-~7A#{TmM8_it{(|NWa=!hiqf-VO)9_}^Vw|NXnmln(!kgNuR=|9H>o_20icC-D1s zU-WT-*6$zROZ@)v;_>$nQ^LV-?a!y@`}2?A+8=jl|Ne11?e8D=6aD^i z1Ih1;E4aTePV@Y}_`3W1A~*ki5##>8xCQ<9#l2m>FK#&cef3Ju!5`SG&q2Sh`ikFI zciR5Gxor6R=9', '.']) + + def test_sequence_builders(self): + tokenizer = AlbertTokenizer(SAMPLE_VOCAB) + + text = tokenizer.encode("sequence builders") + text_2 = tokenizer.encode("multi-sequence build") + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_2 + [tokenizer.sep_token_id] + + +if __name__ == '__main__': + unittest.main() diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index f2e37222f6..0785e55ad2 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -8,6 +8,7 @@ from shutil import copyfile logger = logging.getLogger(__name__) +VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'} SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): @@ -16,12 +17,12 @@ class AlbertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ - # vocab_files_names = VOCAB_FILES_NAMES + vocab_files_names = VOCAB_FILES_NAMES # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, - do_lower_case=False, remove_space=True, keep_accents=False, + do_lower_case=True, remove_space=True, keep_accents=False, bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, @@ -142,15 +143,15 @@ class AlbertTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An ALBERT sequence has the following format: + single sequence: [CLS] X [SEP] + pair of sequences: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] if token_ids_1 is None: - return token_ids_0 + sep + cls - return token_ids_0 + sep + token_ids_1 + sep + cls + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): """ @@ -175,25 +176,24 @@ class AlbertTokenizer(PreTrainedTokenizer): return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) if token_ids_1 is not None: - return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1] - return ([0] * len(token_ids_0)) + [1, 1] + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: - 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 - | first sequence | second sequence | CLS segment ID + An ALBERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence if token_ids_1 is None, only returns the first portion of the mask (0's). """ sep = [self.sep_token_id] cls = [self.cls_token_id] - cls_segment_id = [2] if token_ids_1 is None: - return len(token_ids_0 + sep + cls) * [0] - return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index a4f1a6e3ba..c01fbbbeeb 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -185,9 +185,9 @@ class XLNetTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An XLNet sequence has the following format: + single sequence: X + pair of sequences: A B """ sep = [self.sep_token_id] cls = [self.cls_token_id] @@ -224,7 +224,7 @@ class XLNetTokenizer(PreTrainedTokenizer): def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: + An XLNet sequence pair mask has the following format: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 | first sequence | second sequence | CLS segment ID From 1e5b31c3881cb1313216ce0f3cffae89b0845d4f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:25:32 +0000 Subject: [PATCH 32/91] Several fixes and improvements --- transformers/modeling_albert.py | 36 +++++++++--------- .../{30k-clean.model => spiece.model} | Bin transformers/tokenization_albert.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) rename transformers/tests/fixtures/{30k-clean.model => spiece.model} (100%) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index ad8b979cef..371a2e535c 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -7,6 +7,7 @@ import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from transformers.modeling_utils import PreTrainedModel from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -37,18 +38,17 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): print(name) for name, array in zip(names, arrays): - print(name) - og = name + original_name = name name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") - name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("LayerNorm_1", "attention/LayerNorm") + name = name.replace("cls/predictions", "predictions") + name = name.replace("transform/", "") + name = name.replace("LayerNorm_1", "full_layer_layer_norm") + name = name.replace("LayerNorm", "attention/LayerNorm") name = name.replace("inner_group_", "albert_layers/") name = name.replace("group_", "albert_layer_groups/") name = name.split('/') - - print(name) pointer = model for m_name in name: if re.fullmatch(r'[A-Za-z]+_\d+', m_name): @@ -78,13 +78,12 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): pointer = getattr(pointer, 'weight') elif m_name == 'kernel': array = np.transpose(array) - print("transposed") try: assert pointer.shape == array.shape except AssertionError as e: e.args += (pointer.shape, array.shape) raise - print("Initialize PyTorch weight {} from {}".format(name, og)) + print("Initialize PyTorch weight {} from {}".format(name, original_name)) pointer.data = torch.from_numpy(array) return model @@ -177,9 +176,9 @@ class AlbertAttention(BertSelfAttention): b = self.dense.bias projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b - projected_context_layer = self.dropout(projected_context_layer) - layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) - return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + projected_context_layer_dropout = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) + return layernormed_context_layer class AlbertLayer(nn.Module): @@ -187,17 +186,17 @@ class AlbertLayer(nn.Module): super(AlbertLayer, self).__init__() self.config = config - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.full_layer_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask)[0] + attention_output = self.attention(hidden_states, attention_mask) ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) return hidden_states @@ -352,16 +351,17 @@ class AlbertModel(BertModel): encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) + sequence_output = encoder_outputs[0] - + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) - outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(nn.Module): +class AlbertForMaskedLM(PreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -384,7 +384,7 @@ class AlbertForMaskedLM(nn.Module): """ def __init__(self, config): - super(AlbertForMaskedLM, self).__init__() + super(AlbertForMaskedLM, self).__init__(config) self.config = config self.bert = AlbertModel(config) diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/spiece.model similarity index 100% rename from transformers/tests/fixtures/30k-clean.model rename to transformers/tests/fixtures/spiece.model diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 0785e55ad2..7b16bb573f 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -8,7 +8,7 @@ from shutil import copyfile logger = logging.getLogger(__name__) -VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'} +VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): From 5680a1106302b1ebeb960de0700d6379c0aeef5c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:42:49 +0000 Subject: [PATCH 33/91] Activation function managed from the config file --- transformers/configuration_albert.py | 2 +- transformers/modeling_albert.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index c86c9565cb..15437dbbea 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -16,7 +16,7 @@ class AlbertConfig(PretrainedConfig): intermediate_size=16384, inner_group_num=1, down_scale_factor=1, - hidden_act="gelu", + hidden_act="gelu_new", hidden_dropout_prob=0, attention_probs_dropout_prob=0, max_position_embeddings=512, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 371a2e535c..7e9f7f1c46 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from transformers.modeling_utils import PreTrainedModel from .file_utils import add_start_docstrings @@ -190,11 +190,12 @@ class AlbertLayer(nn.Module): self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + self.activation = ACT2FN[config.hidden_act] def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_output = self.attention(hidden_states, attention_mask) ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) + ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) @@ -392,6 +393,7 @@ class AlbertForMaskedLM(PreTrainedModel): self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] def tie_weights(self): """ Make sure we are sharing the input and output embeddings. @@ -405,7 +407,7 @@ class AlbertForMaskedLM(PreTrainedModel): outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] hidden_states = self.dense(sequence_outputs) - hidden_states = gelu_new(hidden_states) + hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) prediction_scores = self.word_embeddings(hidden_states) From ce9eade29c75fa676ac528d1fe21d9f4ac3c5622 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:50:44 +0000 Subject: [PATCH 34/91] Initializer range using BertPreTrainedModel --- transformers/modeling_albert.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 7e9f7f1c46..b45208b696 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -6,8 +6,7 @@ import torch import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN -from transformers.modeling_utils import PreTrainedModel +from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -362,7 +361,7 @@ class AlbertModel(BertModel): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(PreTrainedModel): +class AlbertForMaskedLM(BertPreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. From 25a31953e8820eb5c88d8ad35ee547efccfe577c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 21:18:06 +0000 Subject: [PATCH 35/91] Output Attentions + output hidden states --- transformers/modeling_albert.py | 58 ++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b45208b696..52cab2ea69 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -105,6 +105,7 @@ class AlbertAttention(BertSelfAttention): def __init__(self, config): super(AlbertAttention, self).__init__(config) + self.output_attentions = config.output_attentions self.num_attention_heads = config.num_attention_heads self.hidden_size = config.hidden_size self.attention_head_size = config.hidden_size // config.num_attention_heads @@ -177,7 +178,7 @@ class AlbertAttention(BertSelfAttention): projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b projected_context_layer_dropout = self.dropout(projected_context_layer) layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) - return layernormed_context_layer + return (layernormed_context_layer, attention_probs) if self.output_attentions else (layernormed_context_layer,) class AlbertLayer(nn.Module): @@ -193,25 +194,45 @@ class AlbertLayer(nn.Module): def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_output = self.attention(hidden_states, attention_mask) - ffn_output = self.ffn(attention_output) + ffn_output = self.ffn(attention_output[0]) ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output[0]) - return hidden_states + return (hidden_states,) + attention_output[1:] # add attentions if we output them class AlbertLayerGroup(nn.Module): def __init__(self, config): super(AlbertLayerGroup, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for albert_layer in self.albert_layers: - hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + layer_hidden_states = () + layer_attentions = () - return hidden_states + for albert_layer in self.albert_layers: + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + layer_output = albert_layer(hidden_states, attention_mask, head_mask) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + return outputs # last-layer hidden state, (layer hidden states), (layer attentions) class AlbertTransformer(nn.Module): @@ -227,11 +248,30 @@ class AlbertTransformer(nn.Module): def forward(self, hidden_states, attention_mask=None, head_mask=None): hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + for layer_idx in range(self.config.num_hidden_layers): group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) - return (hidden_states,) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in From 870320a24e28f187d0dfd10b82c4d60d5269374d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 22:30:21 +0000 Subject: [PATCH 36/91] Early tests --- transformers/modeling_albert.py | 59 +++---- transformers/tests/modeling_albert_test.py | 191 +++++++++++++++++++++ 2 files changed, 219 insertions(+), 31 deletions(-) create mode 100644 transformers/tests/modeling_albert_test.py diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 52cab2ea69..f906352311 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -11,6 +11,15 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) + +ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", +} + + def load_tf_weights_in_albert(model, config, tf_checkpoint_path): """ Load tf checkpoints in a pytorch model.""" try: @@ -39,6 +48,7 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): for name, array in zip(names, arrays): original_name = name name = name.replace("ffn_1", "ffn") + name = name.replace("/bert/", "/albert/") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions", "predictions") @@ -114,29 +124,6 @@ class AlbertAttention(BertSelfAttention): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.pruned_heads = set() - def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.num_attention_heads, self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - - # Prune linear layers - self.query = prune_linear_layer(self.query, index) - self.key = prune_linear_layer(self.key, index) - self.value = prune_linear_layer(self.value, index) - self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) - - # Update hyper params and store pruned heads - self.num_attention_heads = self.num_attention_heads - len(heads) - self.all_head_size = self.attention_head_size * self.num_attention_heads - self.pruned_heads = self.pruned_heads.union(heads) - def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) mixed_key_layer = self.key(input_ids) @@ -225,7 +212,7 @@ class AlbertLayerGroup(nn.Module): layer_attentions = layer_attentions + (layer_output[1],) if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) + layer_hidden_states = layer_hidden_states + (hidden_states,) outputs = (hidden_states,) if self.output_hidden_states: @@ -367,6 +354,8 @@ class AlbertModel(BertModel): self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() + self.init_weights() + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: @@ -422,33 +411,41 @@ class AlbertForMaskedLM(BertPreTrainedModel): list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) self.config = config - self.bert = AlbertModel(config) + self.albert = AlbertModel(config) self.LayerNorm = nn.LayerNorm(config.embedding_size) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) - self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) self.activation = ACT2FN[config.hidden_act] + self.init_weights() + self.tie_weights() + def tie_weights(self): """ Make sure we are sharing the input and output embeddings. Export to TorchScript can't handle parameter sharing so we are cloning them instead. """ - self._tie_or_clone_weights(self.classifier.word_embeddings, - self.transformer.embeddings.word_embeddings) + self._tie_or_clone_weights(self.decoder, + self.albert.embeddings.word_embeddings) def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): - outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] hidden_states = self.dense(sequence_outputs) hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) - prediction_scores = self.word_embeddings(hidden_states) + prediction_scores = self.decoder(hidden_states) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here if masked_lm_labels is not None: diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py new file mode 100644 index 0000000000..46a2eeb729 --- /dev/null +++ b/transformers/tests/modeling_albert_test.py @@ -0,0 +1,191 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +from transformers import is_torch_available + +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +if is_torch_available(): + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM) + from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP +else: + pytestmark = pytest.mark.skip("Require Torch") + + +class AlbertModelTest(CommonTestCases.CommonModelTester): + + all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () + test_pruning = False + test_head_masking = False + + class AlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertModel(config=config) + model.eval() + sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output, + "pooled_output": pooled_output, + } + self.parent.assertListEqual( + list(result["sequence_output"].size()), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size]) + + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForMaskedLM(config=config) + model.eval() + loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) + result = { + "loss": loss, + "prediction_scores": prediction_scores, + } + self.parent.assertListEqual( + list(result["prediction_scores"].size()), + [self.batch_size, self.seq_length, self.vocab_size]) + self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = AlbertModelTest.AlbertModelTester(self) + self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = AlbertModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From c14a22272f3fc17bb2eaeca62986c31a7d26bc85 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 14:04:10 +0000 Subject: [PATCH 37/91] ALBERT passes all tests --- transformers/configuration_albert.py | 4 +--- transformers/modeling_albert.py | 9 +++------ transformers/tests/tokenization_albert_test.py | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index 15437dbbea..04f9fa8d60 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -7,7 +7,7 @@ class AlbertConfig(PretrainedConfig): """ def __init__(self, - vocab_size_or_config_json_file, + vocab_size_or_config_json_file=30000, embedding_size=128, hidden_size=4096, num_hidden_layers=12, @@ -15,7 +15,6 @@ class AlbertConfig(PretrainedConfig): num_attention_heads=64, intermediate_size=16384, inner_group_num=1, - down_scale_factor=1, hidden_act="gelu_new", hidden_dropout_prob=0, attention_probs_dropout_prob=0, @@ -61,7 +60,6 @@ class AlbertConfig(PretrainedConfig): self.num_hidden_groups = num_hidden_groups self.num_attention_heads = num_attention_heads self.inner_group_num = inner_group_num - self.down_scale_factor = down_scale_factor self.hidden_act = hidden_act self.intermediate_size = intermediate_size self.hidden_dropout_prob = hidden_dropout_prob diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index f906352311..9bb38dead9 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -202,17 +202,14 @@ class AlbertLayerGroup(nn.Module): layer_attentions = () for albert_layer in self.albert_layers: - if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) - layer_output = albert_layer(hidden_states, attention_mask, head_mask) hidden_states = layer_output[0] if self.output_attentions: layer_attentions = layer_attentions + (layer_output[1],) - if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) outputs = (hidden_states,) if self.output_hidden_states: @@ -247,7 +244,7 @@ class AlbertTransformer(nn.Module): hidden_states = layer_group_output[0] if self.output_attentions: - all_attentions = all_attentions + layer_group_output[1] + all_attentions = all_attentions + layer_group_output[-1] if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) diff --git a/transformers/tests/tokenization_albert_test.py b/transformers/tests/tokenization_albert_test.py index dd63f6756b..59eb3bceb0 100644 --- a/transformers/tests/tokenization_albert_test.py +++ b/transformers/tests/tokenization_albert_test.py @@ -22,7 +22,7 @@ from transformers.tokenization_albert import (AlbertTokenizer, SPIECE_UNDERLINE) from .tokenization_tests_commons import CommonTestCases SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'fixtures/30k-clean.model') + 'fixtures/spiece.model') class AlbertTokenizationTest(CommonTestCases.CommonTokenizerTester): From b21402fc86257feca05ab050d061e15441a49929 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 14:19:31 +0000 Subject: [PATCH 38/91] Python 2 tests + licence --- transformers/modeling_albert.py | 16 ++++++++++++++++ transformers/tokenization_albert.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 9bb38dead9..b6d1291725 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -1,4 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch ALBERT model. """ + import os import math import logging diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 7b16bb573f..7cba99b9e4 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -1,4 +1,21 @@ - +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization classes for ALBERT model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + from .tokenization_utils import PreTrainedTokenizer import logging import unicodedata From c4403006b8301a24bfeec99c39ce8d5d47df570f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 15:30:11 +0000 Subject: [PATCH 39/91] External MLM head --- transformers/configuration_albert.py | 17 ++++++++++++++ transformers/modeling_albert.py | 35 +++++++++++++++++++--------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index 04f9fa8d60..b72bbb971e 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -1,3 +1,20 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ALBERT model configuration """ + from .configuration_utils import PretrainedConfig class AlbertConfig(PretrainedConfig): diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b6d1291725..487455e561 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -401,6 +401,26 @@ class AlbertModel(BertModel): outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs +class AlbertMLMHead(nn.Module): + def __init__(self, config): + super(AlbertMLMHead, self).__init__() + + self.LayerNorm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.decoder(hidden_states) + + prediction_scores = hidden_states + self.bias + + return prediction_scores + @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(BertPreTrainedModel): @@ -433,13 +453,8 @@ class AlbertForMaskedLM(BertPreTrainedModel): def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) - self.config = config self.albert = AlbertModel(config) - self.LayerNorm = nn.LayerNorm(config.embedding_size) - self.bias = nn.Parameter(torch.zeros(config.vocab_size)) - self.dense = nn.Linear(config.hidden_size, config.embedding_size) - self.decoder = nn.Linear(config.embedding_size, config.vocab_size) - self.activation = ACT2FN[config.hidden_act] + self.predictions = AlbertMLMHead(config) self.init_weights() self.tie_weights() @@ -448,17 +463,15 @@ class AlbertForMaskedLM(BertPreTrainedModel): """ Make sure we are sharing the input and output embeddings. Export to TorchScript can't handle parameter sharing so we are cloning them instead. """ - self._tie_or_clone_weights(self.decoder, + self._tie_or_clone_weights(self.predictions.decoder, self.albert.embeddings.word_embeddings) def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] - hidden_states = self.dense(sequence_outputs) - hidden_states = self.activation(hidden_states) - hidden_states = self.LayerNorm(hidden_states) - prediction_scores = self.decoder(hidden_states) + + prediction_scores = self.predictions(sequence_outputs) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here if masked_lm_labels is not None: From 4f3a54bfc8fa8749f6d5b29f110148738a646fcd Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 16:37:34 +0000 Subject: [PATCH 40/91] ALBERT can load pre-trained models. Doesn't inherit from BERT anymore. --- transformers/__init__.py | 2 +- transformers/configuration_albert.py | 9 ++++++ transformers/modeling_albert.py | 44 +++++++++++++++++++++++----- transformers/tokenization_albert.py | 25 +++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 152d520e7b..bdfb1a0922 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index b72bbb971e..c35426768f 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -17,12 +17,21 @@ from .configuration_utils import PretrainedConfig +ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", +} + class AlbertConfig(PretrainedConfig): """Configuration for `AlbertModel`. The default settings match the configuration of model `albert_xxlarge`. """ + pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP + def __init__(self, vocab_size_or_config_json_file=30000, embedding_size=128, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 487455e561..4da10ed1cb 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -21,6 +21,7 @@ import logging import torch import torch.nn as nn from torch.nn import CrossEntropyLoss +from transformers.modeling_utils import PreTrainedModel from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings @@ -274,6 +275,29 @@ class AlbertTransformer(nn.Module): return outputs # last-layer hidden state, (all hidden states), (all attentions) + +class AlbertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + def _init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if isinstance(module, (nn.Linear)) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents @@ -338,7 +362,7 @@ ALBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertModel(BertModel): +class AlbertModel(AlbertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` @@ -358,6 +382,12 @@ class AlbertModel(BertModel): list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -369,6 +399,11 @@ class AlbertModel(BertModel): self.init_weights() + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: @@ -423,7 +458,7 @@ class AlbertMLMHead(nn.Module): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(BertPreTrainedModel): +class AlbertForMaskedLM(AlbertPreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -445,11 +480,6 @@ class AlbertForMaskedLM(BertPreTrainedModel): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - config_class = AlbertConfig - pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP - load_tf_weights = load_tf_weights_in_albert - base_model_prefix = "albert" - def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 7cba99b9e4..acf67c1154 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -15,7 +15,7 @@ """ Tokenization classes for ALBERT model.""" from __future__ import (absolute_import, division, print_function, unicode_literals) - + from .tokenization_utils import PreTrainedTokenizer import logging import unicodedata @@ -24,8 +24,25 @@ import os from shutil import copyfile logger = logging.getLogger(__name__) - VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'albert-base': 512, + 'albert-large': 512, + 'albert-xlarge': 512, + 'albert-xxlarge': 512, +} + SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): @@ -35,8 +52,8 @@ class AlbertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ vocab_files_names = VOCAB_FILES_NAMES - # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, do_lower_case=True, remove_space=True, keep_accents=False, From c9875455929a63f12b81e2dcc8fe30f72137c06b Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 18:48:02 +0000 Subject: [PATCH 41/91] Converting script --- transformers/__init__.py | 2 +- ...lbert_original_tf_checkpoint_to_pytorch.py | 36 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index bdfb1a0922..db98d5fd44 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py index 04877d41b9..5bbaab8c21 100644 --- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -1,18 +1,39 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert ALBERT checkpoint.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import argparse +import torch -from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert - +from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_albert +import logging +logging.basicConfig(level=logging.INFO) def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # Initialise PyTorch model - config = BertConfig.from_json_file(bert_config_file) + config = AlbertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) - model = BertForPreTraining(config) + model = AlbertForMaskedLM(config) # Load weights from tf checkpoint - load_tf_weights_in_bert(model, config, tf_checkpoint_path) + load_tf_weights_in_albert(model, config, tf_checkpoint_path) # Save pytorch-model print("Save PyTorch model to {}".format(pytorch_dump_path)) @@ -31,7 +52,7 @@ if __name__ == "__main__": default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained BERT model. \n" + help = "The config json file corresponding to the pre-trained ALBERT model. \n" "This specifies the model architecture.") parser.add_argument("--pytorch_dump_path", default = None, @@ -40,5 +61,6 @@ if __name__ == "__main__": help = "Path to the output PyTorch model.") args = parser.parse_args() convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, - args.bert_config_file, + args.albert_config_file, args.pytorch_dump_path) + \ No newline at end of file From 0d07a23c04c9837234cda402b32246d7581e5bc4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 15:07:01 +0000 Subject: [PATCH 42/91] LAMB implementation --- transformers/optimization.py | 93 ++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/transformers/optimization.py b/transformers/optimization.py index 99e6cc75e4..90f3dbca9b 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -167,3 +167,96 @@ class AdamW(Optimizer): p.data.add_(-group['lr'] * group['weight_decay'], p.data) return loss + + + +class Lamb(Optimizer): + """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training). + + Adapted from the huggingface/transformers ADAM optimizer + Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py + Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py + + + Parameters: + lr (float): learning rate. Default 1e-3. + betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) + eps (float): Adams epsilon. Default: 1e-6 + weight_decay (float): Weight decay. Default: 0.0 + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): + if lr < 0.0: + raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, + correct_bias=correct_bias) + super(Lamb, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('LAMB does not support sparse gradients.') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + # Decay the first and second moment running average coefficient + # In-place operations to update the averages at the same time + exp_avg.mul_(beta1).add_(1.0 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) + denom = exp_avg_sq.sqrt().add_(group['eps']) + + + # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py + step_size = group['lr'] + weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) + + adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) + if group['weight_decay'] != 0: + adam_step.add_(group['weight_decay'], p.data) + + adam_norm = adam_step.pow(2).sum().sqrt() + if weight_norm == 0 or adam_norm == 0: + trust_ratio = 1 + else: + trust_ratio = weight_norm / adam_norm + + + state['weight_norm'] = weight_norm + state['adam_norm'] = adam_norm + state['trust_ratio'] = trust_ratio + + p.data.add_(-step_size * trust_ratio, adam_step) + return loss From 6637a77f807615ef2427c0390a015f4eb4814fb4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 15:17:31 +0000 Subject: [PATCH 43/91] AlbertForSequenceClassification --- transformers/__init__.py | 3 +- transformers/modeling_albert.py | 77 ++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index db98d5fd44..51995942ce 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,8 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 4da10ed1cb..bba6767079 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -20,10 +20,10 @@ import math import logging import torch import torch.nn as nn -from torch.nn import CrossEntropyLoss +from torch.nn import CrossEntropyLoss, MSELoss from transformers.modeling_utils import PreTrainedModel from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN +from transformers.modeling_bert import BertEmbeddings, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -510,3 +510,76 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): outputs = (masked_lm_loss,) + outputs return outputs + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForSequenceClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base') + model = AlbertForSequenceClassification.from_pretrained('albert-base') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(AlbertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.albert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), logits, (hidden_states), (attentions) \ No newline at end of file From c110c41fdb3b363528336ceda3b9fb46f026ad9c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 21:59:40 +0000 Subject: [PATCH 44/91] Run GLUE and remove LAMB --- examples/run_glue.py | 14 ++++-- transformers/optimization.py | 93 ------------------------------------ 2 files changed, 10 insertions(+), 97 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 527e440075..550a0b8175 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -47,7 +47,11 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForSequenceClassification, - DistilBertTokenizer) + DistilBertTokenizer, + AlbertConfig, + AlbertForSequenceClassification, + AlbertTokenizer, + ) from transformers import AdamW, get_linear_schedule_with_warmup @@ -66,7 +70,8 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer) } @@ -99,6 +104,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: @@ -317,7 +323,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): all_labels = torch.tensor([f.label for f in features], dtype=torch.long) elif output_mode == "regression": all_labels = torch.tensor([f.label for f in features], dtype=torch.float) - + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset @@ -361,7 +367,7 @@ def main(): parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.") + help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, diff --git a/transformers/optimization.py b/transformers/optimization.py index 90f3dbca9b..99e6cc75e4 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -167,96 +167,3 @@ class AdamW(Optimizer): p.data.add_(-group['lr'] * group['weight_decay'], p.data) return loss - - - -class Lamb(Optimizer): - """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training). - - Adapted from the huggingface/transformers ADAM optimizer - Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py - Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py - - - Parameters: - lr (float): learning rate. Default 1e-3. - betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) - eps (float): Adams epsilon. Default: 1e-6 - weight_decay (float): Weight decay. Default: 0.0 - """ - - def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): - if lr < 0.0: - raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) - if not 0.0 <= betas[0] < 1.0: - raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) - if not 0.0 <= betas[1] < 1.0: - raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) - if not 0.0 <= eps: - raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) - defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, - correct_bias=correct_bias) - super(Lamb, self).__init__(params, defaults) - - def step(self, closure=None): - """Performs a single optimization step. - - Arguments: - closure (callable, optional): A closure that reevaluates the model - and returns the loss. - """ - loss = None - if closure is not None: - loss = closure() - - for group in self.param_groups: - for p in group['params']: - if p.grad is None: - continue - grad = p.grad.data - if grad.is_sparse: - raise RuntimeError('LAMB does not support sparse gradients.') - - state = self.state[p] - - # State initialization - if len(state) == 0: - state['step'] = 0 - # Exponential moving average of gradient values - state['exp_avg'] = torch.zeros_like(p.data) - # Exponential moving average of squared gradient values - state['exp_avg_sq'] = torch.zeros_like(p.data) - - exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] - beta1, beta2 = group['betas'] - - state['step'] += 1 - - # Decay the first and second moment running average coefficient - # In-place operations to update the averages at the same time - exp_avg.mul_(beta1).add_(1.0 - beta1, grad) - exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) - denom = exp_avg_sq.sqrt().add_(group['eps']) - - - # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py - step_size = group['lr'] - weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) - - adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) - if group['weight_decay'] != 0: - adam_step.add_(group['weight_decay'], p.data) - - adam_norm = adam_step.pow(2).sum().sqrt() - if weight_norm == 0 or adam_norm == 0: - trust_ratio = 1 - else: - trust_ratio = weight_norm / adam_norm - - - state['weight_norm'] = weight_norm - state['adam_norm'] = adam_norm - state['trust_ratio'] = trust_ratio - - p.data.add_(-step_size * trust_ratio, adam_step) - return loss From 70d99980ded1565c9e8efa2cd04c21572b664f2f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 4 Nov 2019 11:34:30 -0500 Subject: [PATCH 45/91] ALBERT-V2 --- transformers/configuration_albert.py | 12 ++++++---- ...lbert_original_tf_checkpoint_to_pytorch.py | 5 ++-- transformers/modeling_albert.py | 16 ++++++++----- transformers/tokenization_albert.py | 24 ++++++++++++------- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index c35426768f..de665c9b1c 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -18,10 +18,14 @@ from .configuration_utils import PretrainedConfig ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-config.json", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-config.json", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-config.json", } class AlbertConfig(PretrainedConfig): diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py index 5bbaab8c21..b6476b4fb6 100644 --- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -26,9 +26,10 @@ from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_alb import logging logging.basicConfig(level=logging.INFO) -def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_file, pytorch_dump_path): # Initialise PyTorch model - config = AlbertConfig.from_json_file(bert_config_file) + config = AlbertConfig.from_json_file(albert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = AlbertForMaskedLM(config) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index bba6767079..51cb0a6d23 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -30,10 +30,14 @@ logger = logging.getLogger(__name__) ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-pytorch_model.bin", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-pytorch_model.bin", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-pytorch_model.bin", } @@ -538,8 +542,8 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): Examples:: - tokenizer = AlbertTokenizer.from_pretrained('albert-base') - model = AlbertForSequenceClassification.from_pretrained('albert-base') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForSequenceClassification.from_pretrained('albert-base-v2') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index acf67c1154..2f9af0b0bc 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -29,18 +29,26 @@ VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-spiece.model", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-spiece.model", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-spiece.model", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - 'albert-base': 512, - 'albert-large': 512, - 'albert-xlarge': 512, - 'albert-xxlarge': 512, + 'albert-base-v1': 512, + 'albert-large-v1': 512, + 'albert-xlarge-v1': 512, + 'albert-xxlarge-v1': 512, + 'albert-base-v2': 512, + 'albert-large-v2': 512, + 'albert-xlarge-v2': 512, + 'albert-xxlarge-v2': 512, } SPIECE_UNDERLINE = u'▁' From 4374eaea786399a948b4b34d39fc614ded5e1de6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 6 Nov 2019 20:47:42 +0000 Subject: [PATCH 46/91] ALBERT for SQuAD --- examples/run_squad.py | 12 +++-- transformers/__init__.py | 1 + transformers/modeling_albert.py | 93 ++++++++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 6 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 69088d73c3..59683c0668 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -43,7 +43,8 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLMTokenizer, XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer, - DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer, + AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) from transformers import AdamW, get_linear_schedule_with_warmup @@ -65,7 +66,8 @@ MODEL_CLASSES = { 'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) } def set_seed(args): @@ -128,7 +130,7 @@ def train(args, train_dataset, model, tokenizer): logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) logger.info(" Total optimization steps = %d", t_total) - global_step = 0 + global_step = 1 tr_loss, logging_loss = 0.0, 0.0 model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) @@ -537,7 +539,7 @@ def main(): torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) # Load a trained model and vocabulary that you have fine-tuned - model = model_class.from_pretrained(args.output_dir) + model = model_class.from_pretrained(args.output_dir, force_download=True) tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) model.to(args.device) @@ -555,7 +557,7 @@ def main(): for checkpoint in checkpoints: # Reload the model global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" - model = model_class.from_pretrained(checkpoint) + model = model_class.from_pretrained(checkpoint, force_download=True) model.to(args.device) # Evaluate diff --git a/transformers/__init__.py b/transformers/__init__.py index 51995942ce..81e659329d 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -108,6 +108,7 @@ if is_torch_available(): from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + AlbertForQuestionAnswering, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 51cb0a6d23..2540218e69 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -586,4 +586,95 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) outputs = (loss,) + outputs - return outputs # (loss), logits, (hidden_states), (attentions) \ No newline at end of file + return outputs # (loss), logits, (hidden_states), (attentions) + + + +@add_start_docstrings("""Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForQuestionAnswering(AlbertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForQuestionAnswering.from_pretrained('albert-base-v2') + question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]" + input_ids = tokenizer.encode(input_text) + token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] + start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids])) + all_tokens = tokenizer.convert_ids_to_tokens(input_ids) + print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])) + # a nice puppet + + + """ + def __init__(self, config): + super(AlbertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.albert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) From abb23a78bab17ec09dde4635c32f4aa21c15fa83 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:09:16 +0000 Subject: [PATCH 47/91] Head pruning for ALBERT --- transformers/modeling_albert.py | 42 ++++++++++++++++++++++ transformers/tests/modeling_albert_test.py | 12 ++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 2540218e69..89ece4b61e 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -145,6 +145,29 @@ class AlbertAttention(BertSelfAttention): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.pruned_heads = set() + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.dense = prune_linear_layer(self.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) mixed_key_layer = self.key(input_ids) @@ -409,6 +432,25 @@ class AlbertModel(AlbertPreTrainedModel): self.embeddings.word_embeddings = new_embeddings return self.embeddings.word_embeddings + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + ALBERT has a different architecture in that its layers are shared across groups, which then has inner groups. + If an ALBERT model has 12 hidden layers and 2 hidden groups, with two inner groups, there + is a total of 4 different layers. + + These layers are flattened: the indices [0,1] correspond to the two inner groups of the first hidden layer, + while [2,3] correspond to the two inner groups of the second hidden layer. + + Any layer with in index other than [0,1,2,3] will result in an error. + See base class PreTrainedModel for more information about head pruning + """ + for layer, heads in heads_to_prune.items(): + group_idx = int(layer / self.config.inner_group_num) + inner_group_idx = int(layer - group_idx * self.config.inner_group_num) + self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 46a2eeb729..979e0488eb 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -35,7 +35,6 @@ else: class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () - test_pruning = False test_head_masking = False class AlbertModelTester(object): @@ -49,9 +48,10 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, - hidden_size=32, - num_hidden_layers=5, - num_attention_heads=4, + hidden_size=36, + num_hidden_layers=6, + num_hidden_groups=6, + num_attention_heads=6, intermediate_size=37, hidden_act="gelu", hidden_dropout_prob=0.1, @@ -86,6 +86,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): self.num_labels = num_labels self.num_choices = num_choices self.scope = scope + self.num_hidden_groups = num_hidden_groups def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) @@ -117,7 +118,8 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): attention_probs_dropout_prob=self.attention_probs_dropout_prob, max_position_embeddings=self.max_position_embeddings, type_vocab_size=self.type_vocab_size, - initializer_range=self.initializer_range) + initializer_range=self.initializer_range, + num_hidden_groups=self.num_hidden_groups) return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels From 16263f9685eaf459408f33c9790a967012b93fa5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:29:29 +0000 Subject: [PATCH 48/91] Headmasking --- transformers/modeling_albert.py | 11 ++++++----- transformers/tests/modeling_albert_test.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 89ece4b61e..6682930d89 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -224,7 +224,7 @@ class AlbertLayer(nn.Module): self.activation = ACT2FN[config.hidden_act] def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask) + attention_output = self.attention(hidden_states, attention_mask, head_mask) ffn_output = self.ffn(attention_output[0]) ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) @@ -245,8 +245,8 @@ class AlbertLayerGroup(nn.Module): layer_hidden_states = () layer_attentions = () - for albert_layer in self.albert_layers: - layer_output = albert_layer(hidden_states, attention_mask, head_mask) + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer(hidden_states, attention_mask, head_mask[layer_index]) hidden_states = layer_output[0] if self.output_attentions: @@ -283,7 +283,8 @@ class AlbertTransformer(nn.Module): for layer_idx in range(self.config.num_hidden_layers): group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) hidden_states = layer_group_output[0] @@ -544,7 +545,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): - outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask) sequence_outputs = outputs[0] prediction_scores = self.predictions(sequence_outputs) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 979e0488eb..466f473332 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -35,7 +35,6 @@ else: class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () - test_head_masking = False class AlbertModelTester(object): From 9d5c49546fedb3d52f76f97f4043a07e08ded918 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:32:52 +0000 Subject: [PATCH 49/91] Tests for AlbertForQuestionAnswering AlbertForSequenceClassification --- transformers/tests/modeling_albert_test.py | 45 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 466f473332..da87709df1 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -26,7 +26,9 @@ from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester if is_torch_available(): - from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM) + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM, + AlbertForSequenceClassification, AlbertForQuestionAnswering, + ) from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -157,6 +159,39 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): [self.batch_size, self.seq_length, self.vocab_size]) self.check_loss_output(result) + def create_and_check_albert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForQuestionAnswering(config=config) + model.eval() + loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, + start_positions=sequence_labels, end_positions=sequence_labels) + result = { + "loss": loss, + "start_logits": start_logits, + "end_logits": end_logits, + } + self.parent.assertListEqual( + list(result["start_logits"].size()), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].size()), + [self.batch_size, self.seq_length]) + self.check_loss_output(result) + + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = AlbertForSequenceClassification(config) + model.eval() + loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) + result = { + "loss": loss, + "logits": logits, + } + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.num_labels]) + self.check_loss_output(result) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() @@ -180,6 +215,14 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification(*config_and_inputs) + @pytest.mark.slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" From d9daad98c744e115bfb425f316a5e7d4f405a9a5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 19:55:43 +0000 Subject: [PATCH 50/91] Re-ordering of group_idx/layer_idx + Python 2 tests --- transformers/modeling_albert.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 6682930d89..640af537d0 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -281,11 +281,17 @@ class AlbertTransformer(nn.Module): if self.output_hidden_states: all_hidden_states = (hidden_states,) - for layer_idx in range(self.config.num_hidden_layers): - group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + # Index of the layer inside the group + layer_idx = int(i - group_idx * layers_per_group) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) - hidden_states = layer_group_output[0] if self.output_attentions: From f6f382532bf40a5869dc14e3eefa451646c19ded Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 23:40:45 +0000 Subject: [PATCH 51/91] ALBERT in TF2 --- transformers/__init__.py | 6 +- .../convert_pytorch_checkpoint_to_tf2.py | 13 +- transformers/modeling_tf_albert.py | 723 ++++++++++++++++++ 3 files changed, 736 insertions(+), 6 deletions(-) create mode 100644 transformers/modeling_tf_albert.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 81e659329d..a409ef772e 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -58,8 +58,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_albert import AlbertConfig, ALBERT -from .configuration_albert import AlbertConfig +from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling @@ -169,6 +168,9 @@ if is_tf_available(): TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + # TF 2.0 <=> PyTorch conversion utilities from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, load_pytorch_checkpoint_in_tf2_model, diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index e673b77dcc..d1776e9c14 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -33,7 +33,8 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + AlbertConfig, TFAlbertForMaskedLM, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch @@ -46,7 +47,8 @@ if is_torch_available(): OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) else: (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, @@ -56,7 +58,8 @@ else: OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) = ( + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) = ( None, None, None, None, None, None, None, None, @@ -65,6 +68,7 @@ else: None, None, None, None, None, None, None, None, + None, None, None, None) @@ -85,7 +89,8 @@ MODEL_CLASSES = { 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), - 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'albert': (AlbertConfig, TFAlbertForMaskedLM, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True): diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py new file mode 100644 index 0000000000..8861b7add8 --- /dev/null +++ b/transformers/modeling_tf_albert.py @@ -0,0 +1,723 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 ALBERT model. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_albert import AlbertConfig +from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_bert import ACT2FN, TFBertSelfAttention +from .file_utils import add_start_docstrings + +import logging + +logger = logging.getLogger(__name__) + +TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + # TODO FILL THAT UP +} + + +class TFAlbertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config, **kwargs): + super(TFAlbertEmbeddings, self).__init__(**kwargs) + + self.config = config + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.config.vocab_size, self.config.embedding_size], + initializer=get_initializer(self.config.initializer_range)) + super(TFAlbertEmbeddings, self).build(input_shape) + + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + words_embeddings = tf.gather(self.word_embeddings, input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings, training=training) + return embeddings + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, embedding_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + print(inputs.shape) + + x = tf.reshape(inputs, [-1, self.config.embedding_size]) + + print(x.shape, self.word_embeddings) + + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + + print([batch_size, length, self.config.vocab_size]) + return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) + + +class TFAlbertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int( + config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='value') + + self.dropout = tf.keras.layers.Dropout( + config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape( + x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = tf.shape(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFAlbertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + return outputs + + +class TFAlbertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFAlbertAttention(TFBertSelfAttention): + def __init__(self, config, **kwargs): + super(TFAlbertAttention, self).__init__(config, **kwargs) + + self.hidden_size = config.hidden_size + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + batch_size = tf.shape(input_tensor)[0] + mixed_query_layer = self.query(input_tensor) + mixed_key_layer = self.key(input_tensor) + mixed_value_layer = self.value(input_tensor) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + self_outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + + hidden_states = self_outputs[0] + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + attention_output = self.LayerNorm(hidden_states + input_tensor) + + # add attentions if we output them + outputs = (attention_output,) + self_outputs[1:] + return outputs + + +class TFAlbertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayer, self).__init__(**kwargs) + self.attention = TFAlbertAttention(config, name='attention') + + self.ffn = tf.keras.layers.Dense(config.intermediate_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn') + + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.ffn_output = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn_output') + self.full_layer_layer_norm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='full_layer_layer_norm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention( + [hidden_states, attention_mask, head_mask], training=training) + ffn_output = self.ffn(attention_outputs[0]) + ffn_output = self.activation(ffn_output) + ffn_output = self.ffn_output(ffn_output) + + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.full_layer_layer_norm( + ffn_output + attention_outputs[0]) + + # add attentions if we output them + outputs = (hidden_states,) + attention_outputs[1:] + return outputs + + +class TFAlbertLayerGroup(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayerGroup, self).__init__(**kwargs) + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.albert_layers = [TFAlbertLayer(config, name="albert_layers_._{}".format( + i)) for i in range(config.inner_group_num)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + layer_hidden_states = () + layer_attentions = () + + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer( + [hidden_states, attention_mask, head_mask[layer_index]], training=training) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + # last-layer hidden state, (layer hidden states), (layer attentions) + return outputs + + +class TFAlbertTransformer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertTransformer, self).__init__(**kwargs) + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='embedding_hidden_mapping_in') + self.albert_layer_groups = [TFAlbertLayerGroup( + config, name="albert_layer_groups_._{}".format(i)) for i in range(config.num_hidden_groups)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group + layers_per_group = int( + self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int( + i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + layer_group_output = self.albert_layer_groups[group_idx]( + [hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]], training=training) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[-1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + + # last-layer hidden state, (all hidden states), (all attentions) + return outputs + + +class TFAlbertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + +class TFAlbertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, input_embeddings, **kwargs): + super(TFAlbertMLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + + self.dense = tf.keras.layers.Dense(config.embedding_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.input_embeddings = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + super(TFAlbertMLMHead, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias + return hidden_states + + +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, ALBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Albert Model transformer outputing raw hidden-states without any specific head on top.", + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertModel(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Albert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertModel + + tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') + model = TFAlbertModel.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + + def __init__(self, config, **kwargs): + super(TFAlbertModel, self).__init__(config, **kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFAlbertEmbeddings(config, name="embeddings") + self.encoder = TFAlbertTransformer(config, name="encoder") + self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), activation='tanh', name='pooler') + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings( + [input_ids, position_ids, token_type_ids], training=training) + encoder_outputs = self.encoder( + [embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] + # sequence_output, pooled_output, (hidden_states), (attentions) + return outputs + + +@add_start_docstrings("""Albert Model with a `language modeling` head on top. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForMaskedLM + + tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') + model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + prediction_scores = outputs[0] + + """ + + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForMaskedLM, self).__init__(config, *inputs, **kwargs) + + self.albert = TFAlbertModel(config, name='albert') + self.predictions = TFAlbertMLMHead( + config, self.albert.embeddings, name='predictions') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + sequence_output = outputs[0] + prediction_scores = self.predictions( + sequence_output, training=kwargs.get('training', False)) + + # Add hidden states and attention if they are here + outputs = (prediction_scores,) + outputs[2:] + + return outputs # prediction_scores, (hidden_states), (attentions) From 7bddbf5961f35d03db5151a4747085f7a57d7ff7 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 23:50:05 +0000 Subject: [PATCH 52/91] TFAlbertForSequenceClassification --- transformers/modeling_tf_albert.py | 66 ++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 8861b7add8..410d83ff78 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -479,16 +479,15 @@ class TFAlbertMLMHead(tf.keras.layers.Layer): ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in - `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ - by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer - pre-trained using a combination of masked language modeling objective and next sentence prediction - on a large corpus comprising the Toronto Book Corpus and Wikipedia. + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. - .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: - https://arxiv.org/abs/1810.04805 + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model @@ -695,8 +694,8 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): import tensorflow as tf from transformers import AlbertTokenizer, TFAlbertForMaskedLM - tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') - model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForMaskedLM.from_pretrained('albert-base-v2') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -721,3 +720,54 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): outputs = (prediction_scores,) + outputs[2:] return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForSequenceClassification + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForSequenceClassification.from_pretrained('albert-base-v2') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.albert = TFAlbertModel(config, name='albert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) \ No newline at end of file From b18509c2085dce16e655dd86b7ea0129335da4e1 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 8 Nov 2019 00:12:21 +0000 Subject: [PATCH 53/91] Tests for ALBERT in TF2 + fixes --- transformers/__init__.py | 1 + transformers/modeling_tf_albert.py | 18 +- transformers/tests/modeling_tf_albert_test.py | 229 ++++++++++++++++++ 3 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 transformers/tests/modeling_tf_albert_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index a409ef772e..baf430c17b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -169,6 +169,7 @@ if is_tf_available(): TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # TF 2.0 <=> PyTorch conversion utilities diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 410d83ff78..a3f183b192 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -126,16 +126,8 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): """ batch_size = tf.shape(inputs)[0] length = tf.shape(inputs)[1] - - print(inputs.shape) - x = tf.reshape(inputs, [-1, self.config.embedding_size]) - - print(x.shape, self.word_embeddings) - logits = tf.matmul(x, self.word_embeddings, transpose_b=True) - - print([batch_size, length, self.config.vocab_size]) return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) @@ -460,20 +452,24 @@ class TFAlbertMLMHead(tf.keras.layers.Layer): # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. - self.input_embeddings = input_embeddings + self.decoder = input_embeddings def build(self, input_shape): self.bias = self.add_weight(shape=(self.vocab_size,), initializer='zeros', trainable=True, name='bias') + self.decoder_bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='decoder/bias') super(TFAlbertMLMHead, self).build(input_shape) def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) - hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = self.decoder(hidden_states, mode="linear") + self.decoder_bias hidden_states = hidden_states + self.bias return hidden_states @@ -666,7 +662,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): [embedding_output, extended_attention_mask, head_mask], training=training) sequence_output = encoder_outputs[0] - pooled_output = self.pooler(sequence_output) + pooled_output = self.pooler(sequence_output[:, 0]) # add hidden_states and attentions if they are here outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py new file mode 100644 index 0000000000..85fc62f34f --- /dev/null +++ b/transformers/tests/modeling_tf_albert_test.py @@ -0,0 +1,229 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from transformers import AlbertConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from transformers.modeling_tf_albert import (TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = ( + TFAlbertModel, + TFAlbertForMaskedLM, + TFAlbertForSequenceClassification + ) if is_tf_available() else () + + class TFAlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor( + [self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor( + [self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor( + [self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor( + [self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor( + [self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertModel(config=config) + # inputs = {'input_ids': input_ids, + # 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids} + # sequence_output, pooled_output = model(**inputs) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [ + self.batch_size, self.hidden_size]) + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFAlbertForSequenceClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, + 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFAlbertModelTest.TFAlbertModelTester(self) + self.config_tester = ConfigTester( + self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm( + *config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification( + *config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['albert-base-uncased']: + model = TFAlbertModel.from_pretrained( + model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() From c9cb7f8a0fbe784665b00bfdca6bfc54ad10d5f5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 11 Nov 2019 15:12:54 -0500 Subject: [PATCH 54/91] Torch 1.1.0 compatibility + FP16 O1 + TF checkpoints Co-authored-by: wassname --- transformers/modeling_albert.py | 4 ++-- transformers/modeling_tf_albert.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 640af537d0..ff20ca78dc 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -203,8 +203,8 @@ class AlbertAttention(BertSelfAttention): # Should find a better way to do this - w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) - b = self.dense.bias + w = self.dense.weight.t().view(self.num_attention_heads, self.attention_head_size, self.hidden_size).to(context_layer.dtype) + b = self.dense.bias.to(context_layer.dtype) projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b projected_context_layer_dropout = self.dropout(projected_context_layer) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index a3f183b192..ee8712eb28 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -36,7 +36,14 @@ import logging logger = logging.getLogger(__name__) TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - # TODO FILL THAT UP + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-tf_model.h5", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-tf_model.h5", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-tf_model.h5", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-tf_model.h5", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-tf_model.h5", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-tf_model.h5", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-tf_model.h5", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-tf_model.h5", } From f873b55e433e1c9ef364e597ad1d4662db4bc038 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 26 Nov 2019 10:28:41 -0500 Subject: [PATCH 55/91] Warning for ALBERT-v2 models --- transformers/modeling_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d51eefab58..5d51caa951 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -315,6 +315,10 @@ class PreTrainedModel(nn.Module): model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ + if "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path: + logger.warning("There is currently an upstream reproducibility issue with ALBERT v2 models. Please see " + + "https://github.com/google-research/google-research/issues/119 for more information.") + config = kwargs.pop('config', None) state_dict = kwargs.pop('state_dict', None) cache_dir = kwargs.pop('cache_dir', None) From c536c2a4809a51356fd50ff62a80955fea79f790 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 26 Nov 2019 11:22:52 -0500 Subject: [PATCH 56/91] ALBERT Input Embeds --- transformers/modeling_albert.py | 77 ++++++++++++++++++++++-------- transformers/modeling_tf_albert.py | 45 ++++++++++++----- 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index ff20ca78dc..7882356d24 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -433,6 +433,12 @@ class AlbertModel(AlbertPreTrainedModel): self.init_weights() + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + def _resize_token_embeddings(self, new_num_tokens): old_embeddings = self.embeddings.word_embeddings new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) @@ -457,12 +463,24 @@ class AlbertModel(AlbertPreTrainedModel): inner_group_idx = int(layer - group_idx * self.config.inner_group_num) self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None): + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: - attention_mask = torch.ones_like(input_ids) + attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility @@ -477,7 +495,8 @@ class AlbertModel(AlbertPreTrainedModel): else: head_mask = [None] * self.config.num_hidden_layers - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds) encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) @@ -549,9 +568,19 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): self._tie_or_clone_weights(self.predictions.decoder, self.albert.embeddings.word_embeddings) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - masked_lm_labels=None): - outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask) + def get_output_embeddings(self): + return self.predictions.decoder + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None, inputs_embeds=None): + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) sequence_outputs = outputs[0] prediction_scores = self.predictions(sequence_outputs) @@ -609,14 +638,17 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): - outputs = self.albert(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) pooled_output = outputs[1] @@ -692,14 +724,17 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - start_positions=None, end_positions=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None, start_positions=None, end_positions=None): - outputs = self.albert(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) sequence_output = outputs[0] diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index ee8712eb28..ee122205b9 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -107,19 +107,25 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): def _embedding(self, inputs, training=False): """Applies embedding based on inputs tensor.""" - input_ids, position_ids, token_type_ids = inputs + input_ids, position_ids, token_type_ids, inputs_embeds = inputs - seq_length = tf.shape(input_ids)[1] + if input_ids is not None: + input_shape = tf.shape(input_ids) + else: + input_shape = tf.shape(inputs_embeds)[:-1] + + seq_length = input_shape[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) - words_embeddings = tf.gather(self.word_embeddings, input_ids) + if inputs_embeds is None: + inputs_embeds = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = inputs_embeds + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) embeddings = self.dropout(embeddings, training=training) return embeddings @@ -603,6 +609,9 @@ class TFAlbertModel(TFAlbertPreTrainedModel): self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( config.initializer_range), activation='tanh', name='pooler') + def get_input_embeddings(self): + return self.embeddings + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -613,28 +622,39 @@ class TFAlbertModel(TFAlbertPreTrainedModel): """ raise NotImplementedError - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.shape + elif inputs_embeds is not None: + input_shape = inputs_embeds.shape[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) + attention_mask = tf.fill(input_shape, 1) if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] @@ -664,7 +684,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): # head_mask = tf.constant([0] * self.num_hidden_layers) embedding_output = self.embeddings( - [input_ids, position_ids, token_type_ids], training=training) + [input_ids, position_ids, token_type_ids, inputs_embeds], training=training) encoder_outputs = self.encoder( [embedding_output, extended_attention_mask, head_mask], training=training) @@ -712,6 +732,9 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): self.predictions = TFAlbertMLMHead( config, self.albert.embeddings, name='predictions') + def get_output_embeddings(self): + return self.albert.embeddings + def call(self, inputs, **kwargs): outputs = self.albert(inputs, **kwargs) From bdfe21ab242c2175972f53ca6b4af29ada02a2bb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 12:54:36 -0500 Subject: [PATCH 57/91] Change param order for consistency --- transformers/modeling_albert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 7882356d24..5b7b2d3900 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -571,8 +571,8 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def get_output_embeddings(self): return self.predictions.decoder - def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - masked_lm_labels=None, inputs_embeds=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, + masked_lm_labels=None): outputs = self.albert( input_ids=input_ids, attention_mask=attention_mask, From f2f329408db66285fd59e6628ca394381bb7f94e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 12:59:28 -0500 Subject: [PATCH 58/91] Fix input embeddings --- transformers/tests/modeling_albert_test.py | 2 ++ transformers/tests/modeling_tf_albert_test.py | 2 ++ transformers/tests/modeling_tf_common_test.py | 7 ++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index da87709df1..976feff9db 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -49,6 +49,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, + embedding_size=16, hidden_size=36, num_hidden_layers=6, num_hidden_groups=6, @@ -73,6 +74,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index 85fc62f34f..fbd519b8f6 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -54,6 +54,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, + embedding_size=16, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, @@ -77,6 +78,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 2bb7cc9c5f..31a30766cf 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,9 +426,10 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) - # ^^ In our TF models, the input_embeddings can take slightly different forms, - # so we try two of them and fall back to just synthetically creating a dummy tensor of ones. + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From ae98d4599179b299563b679dd33f8a86da12980d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:12:44 -0500 Subject: [PATCH 59/91] Release: v2.2.0 --- .circleci/deploy.sh | 1 + README.md | 2 +- deploy_multi_version_doc.sh | 22 ++++++++++++++++++++++ docs/source/conf.py | 2 +- setup.py | 2 +- transformers/__init__.py | 2 +- 6 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 deploy_multi_version_doc.sh diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 98151963d8..c4b802c9e9 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,3 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 diff --git a/README.md b/README.md index 52e74a6f80..bb6a8eaa1a 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation](https://huggingface.co/transformers/) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more | +| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more | ## Installation diff --git a/deploy_multi_version_doc.sh b/deploy_multi_version_doc.sh new file mode 100644 index 0000000000..bd567213eb --- /dev/null +++ b/deploy_multi_version_doc.sh @@ -0,0 +1,22 @@ +cd docs + +function deploy_doc(){ + echo "Creating doc at commit $1 and pushing to folder $2" + git checkout $1 + if [ ! -z "$2" ] + then + echo "Pushing version" $2 + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 + else + echo "Pushing master" + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir + fi +} + +deploy_doc "master" +deploy_doc "b33a385" v1.0.0 +deploy_doc "fe02e45" v1.1.0 +deploy_doc "89fd345" v1.2.0 +deploy_doc "fc9faa8" v2.0.0 +deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 00c020ab39..f762a89cd2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.1.1' +release = u'2.2.0' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index f49aee68d4..d8dcf7b898 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ from setuptools import find_packages, setup setup( name="transformers", - version="2.1.1", + version="2.2.0", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", diff --git a/transformers/__init__.py b/transformers/__init__.py index baf430c17b..dff2479d4b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.1.1" +__version__ = "2.2.0" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. From b6321452738925983ba1306fb5396982d64f408a Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Tue, 26 Nov 2019 14:27:15 -0500 Subject: [PATCH 60/91] Update master documentation link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb6a8eaa1a..67fb80aecf 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more | +| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation From cf62bdc962c53d9fb7a5820217f1bf844bb6da3b Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 14:37:32 -0500 Subject: [PATCH 61/91] Improve test protocol for inputs_embeds in TF cc @lysandrejik --- transformers/tests/modeling_tf_common_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 31a30766cf..232991915c 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,10 +426,15 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - if hasattr(self.model_tester, "embedding_size"): - x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32) - else: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) + x = wte([input_ids, None, None, None], mode="embedding") + # ^^ In our TF models, the input_embeddings can take slightly different forms, + # so we try a few of them. + # We used to fall back to just synthetically creating a dummy tensor of ones: + # + # if hasattr(self.model_tester, "embedding_size"): + # x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + # else: + # x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From 8742baa53136b906e392fdae57f1c191d1b2370e Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 14:39:47 -0500 Subject: [PATCH 62/91] Improve test protocol for inputs_embeds in TF --- transformers/tests/modeling_tf_common_test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 232991915c..ea8cd1aecd 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,15 +426,17 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - x = wte([input_ids, None, None, None], mode="embedding") + try: + x = wte([input_ids, None, None, None], mode="embedding") + except: + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) # ^^ In our TF models, the input_embeddings can take slightly different forms, # so we try a few of them. # We used to fall back to just synthetically creating a dummy tensor of ones: # - # if hasattr(self.model_tester, "embedding_size"): - # x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) - # else: - # x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From 668aac45d206bc0c8ca95e7a8b565d1ff701da7f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:52:42 -0500 Subject: [PATCH 63/91] Pretrained models --- docs/source/pretrained_models.rst | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index b0a578fd80..d017a3be6c 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -159,5 +159,38 @@ Here is the full list of the currently provided pretrained models together with | | | | CamemBERT using the BERT-base architecture | | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ + .. `__ From 7c6000e412288ddce7d0dbc4f4d9a12f49d28690 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:55:29 -0500 Subject: [PATCH 64/91] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index c4b802c9e9..73960cbe99 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "f2f3294" v2.2.0 +deploy_doc "668aac4" v2.2.0 From ee4647bd5c1343304f7fdf20f06db1fc2524f892 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:10:51 -0500 Subject: [PATCH 65/91] CamemBERT & ALBERT doc --- docs/source/index.rst | 5 ++ docs/source/model_doc/albert.rst | 71 +++++++++++++++++++++++++++++ docs/source/model_doc/camembert.rst | 50 ++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 docs/source/model_doc/albert.rst create mode 100644 docs/source/model_doc/camembert.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 4cd1f48ba8..55ead33b4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -47,6 +47,9 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 6. `XLM `_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `_ by Guillaume Lample and Alexis Conneau. 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. `DistilBERT `_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 `_. +9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. +10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. +11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. .. toctree:: :maxdepth: 2 @@ -89,3 +92,5 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train model_doc/roberta model_doc/distilbert model_doc/ctrl + model_doc/camembert + model_doc/albert diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst new file mode 100644 index 0000000000..cf52f35eb6 --- /dev/null +++ b/docs/source/model_doc/albert.rst @@ -0,0 +1,71 @@ +ALBERT +---------------------------------------------------- + +``AlbrtConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertConfig + :members: + + +``AlbertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertTokenizer + :members: + + +``AlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertModel + :members: + + +``AlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForMaskedLM + :members: + + +``AlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForSequenceClassification + :members: + + +``AlbertForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForQuestionAnswering + :members: + + +``TFAlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertModel + :members: + + +``TFBertForPreTraining`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFBertForPreTraining + :members: + + +``TFAlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForMaskedLM + :members: + + +``TFAlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForSequenceClassification + :members: diff --git a/docs/source/model_doc/camembert.rst b/docs/source/model_doc/camembert.rst new file mode 100644 index 0000000000..82ca9de945 --- /dev/null +++ b/docs/source/model_doc/camembert.rst @@ -0,0 +1,50 @@ +CamemBERT +---------------------------------------------------- + +``CamembertConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertConfig + :members: + + +``CamembertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertTokenizer + :members: + + +``CamembertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertModel + :members: + + +``CamembertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMaskedLM + :members: + + +``CamembertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForSequenceClassification + :members: + + +``CamembertForMultipleChoice`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMultipleChoice + :members: + + +``CamembertForTokenClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForTokenClassification + :members: From 44b82c777f0a4e173fa9c68de630a63fbac1b946 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:15:11 -0500 Subject: [PATCH 66/91] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 73960cbe99..8081153b9f 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "668aac4" v2.2.0 +deploy_doc "ee4647b" v2.2.0 From cf26a0c85e0520d3e3770c9ad8dce94318ea3440 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:40:03 -0500 Subject: [PATCH 67/91] Fix pretrained models table --- README.md | 1 + docs/source/pretrained_models.rst | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 67fb80aecf..dd2c865036 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. +11. **[ALBERT](https://github.com/google-research/google-research/tree/master/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index d017a3be6c..a6bf508d18 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -162,31 +162,31 @@ Here is the full list of the currently provided pretrained models together with | ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | From ce02550d50f2074d54d58b37cbc9845d9a159818 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:47:02 -0500 Subject: [PATCH 68/91] Fix pretrained models table --- docs/source/pretrained_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index a6bf508d18..a312346df7 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -190,7 +190,7 @@ Here is the full list of the currently provided pretrained models together with | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ .. `__ From cc7968227e08858df4a5c618c739e1a3ca050196 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:52:25 -0500 Subject: [PATCH 69/91] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 8081153b9f..61b9f90909 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "ee4647b" v2.2.0 +deploy_doc "ce02550" v2.2.0 From 361620954acf16b27727d763a591257b03f90b5d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 27 Nov 2019 10:11:37 -0500 Subject: [PATCH 70/91] Remove TFBertForPreTraining from ALBERT doc --- docs/source/model_doc/albert.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst index cf52f35eb6..92970c9328 100644 --- a/docs/source/model_doc/albert.rst +++ b/docs/source/model_doc/albert.rst @@ -50,13 +50,6 @@ ALBERT :members: -``TFBertForPreTraining`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: transformers.TFBertForPreTraining - :members: - - ``TFAlbertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 45d767297a78697e4ff75ec3d83bdf35e43e4fff Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 27 Nov 2019 10:12:20 -0500 Subject: [PATCH 71/91] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 61b9f90909..a32581baef 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "ce02550" v2.2.0 +deploy_doc "3616209" v2.2.0 From 88b317739fe56888528c857fc8e90967148a0051 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 27 Nov 2019 09:58:38 +0000 Subject: [PATCH 72/91] Fix issue: #1962, input's shape seem to cause error in 2.2.0 version tf_albert_model --- transformers/modeling_tf_albert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index ee122205b9..b2bf66f750 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -645,7 +645,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: - input_shape = input_ids.shape + input_shape = tf.shape(input_ids) elif inputs_embeds is not None: input_shape = inputs_embeds.shape[:-1] else: From de2696f68e20019fef3a5e1b54de10351abb4145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Tue, 26 Nov 2019 16:30:27 +0100 Subject: [PATCH 73/91] suggest to track repo w/ https rather than ssh --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 136ef8df81..8228dd59d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ Follow these steps to start contributing: ```bash $ git clone git@github.com:/transformers.git $ cd transformers - $ git remote add upstream git@github.com:huggingface/transformers.git + $ git remote add upstream https://github.com/huggingface/transformers.git ``` 3. Create a new branch to hold your development changes: From b5d884d25c96069962030b9e4c9dfb4b1f2f6fa0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 27 Nov 2019 11:05:18 -0500 Subject: [PATCH 74/91] Uniformize #1952 --- README.md | 2 +- examples/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd2c865036..dd20b80590 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Examples are included in the repository but are not shipped with the library. Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: ```bash -git clone git@github.com:huggingface/transformers +git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . ``` diff --git a/examples/README.md b/examples/README.md index 0c57990ea7..e109a12171 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ similar API between the different models. To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: ```bash -git clone git@github.com:huggingface/transformers +git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . ``` From 71f71ddb3e04aa18aaa1e3633a2881493e67b438 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 11:50:42 -0400 Subject: [PATCH 75/91] run_xnli + utils_xnli --- examples/run_xnli.py | 534 +++++++++++++++++++++++++++++++++++++++++ examples/utils_xnli.py | 93 +++++++ 2 files changed, 627 insertions(+) create mode 100644 examples/run_xnli.py create mode 100644 examples/utils_xnli.py diff --git a/examples/run_xnli.py b/examples/run_xnli.py new file mode 100644 index 0000000000..ee37296832 --- /dev/null +++ b/examples/run_xnli.py @@ -0,0 +1,534 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning multi-lingual models on XNLI (Bert, XLM). + Adapted from `examples/run_glue.py`""" + +from __future__ import absolute_import, division, print_function + +import argparse +import glob +import logging +import os +import random + +import numpy as np +import torch +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + TensorDataset) +from torch.utils.data.distributed import DistributedSampler + +try: + from torch.utils.tensorboard import SummaryWriter +except: + from tensorboardX import SummaryWriter + +from tqdm import tqdm, trange + +from transformers import (WEIGHTS_NAME, + BertConfig, BertForSequenceClassification, BertTokenizer, + XLMConfig, XLMForSequenceClassification, XLMTokenizer, + DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + +from transformers import AdamW, WarmupLinearSchedule + +from utils_xnli import xnli_compute_metrics as compute_metrics +from utils_xnli import xnli_output_modes as output_modes +from utils_xnli import xnli_processors as processors + +from transformers import glue_convert_examples_to_features as convert_examples_to_features + +logger = logging.getLogger(__name__) + +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ()) + +MODEL_CLASSES = { + 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), + 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), + # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) +} + + +def set_seed(args): + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.n_gpu > 0: + torch.cuda.manual_seed_all(args.seed) + + +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + if args.local_rank in [-1, 0]: + tb_writer = SummaryWriter() + + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + + if args.max_steps > 0: + t_total = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", t_total) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: + epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) + for step, batch in enumerate(epoch_iterator): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + tr_loss += loss.item() + if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if args.fp16: + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + # Log metrics + if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well + results = evaluate(args, model, tokenizer) + for key, value in results.items(): + tb_writer.add_scalar('eval_{}'.format(key), value, global_step) + tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) + tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + logging_loss = tr_loss + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + if args.tpu: + args.xla_model.optimizer_step(optimizer, barrier=True) + model.zero_grad() + global_step += 1 + + if args.max_steps > 0 and global_step > args.max_steps: + epoch_iterator.close() + break + if args.max_steps > 0 and global_step > args.max_steps: + train_iterator.close() + break + + if args.local_rank in [-1, 0]: + tb_writer.close() + + return global_step, tr_loss / global_step + + +def evaluate(args, model, tokenizer, prefix=""): + eval_task_names = (args.task_name,) + eval_outputs_dirs = (args.output_dir,) + + results = {} + for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): + eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) + + if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: + os.makedirs(eval_output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(eval_dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + out_label_ids = None + for batch in tqdm(eval_dataloader, desc="Evaluating"): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + tmp_eval_loss, logits = outputs[:2] + + eval_loss += tmp_eval_loss.mean().item() + nb_eval_steps += 1 + if preds is None: + preds = logits.detach().cpu().numpy() + out_label_ids = inputs['labels'].detach().cpu().numpy() + else: + preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) + out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0) + + eval_loss = eval_loss / nb_eval_steps + if args.output_mode == "classification": + preds = np.argmax(preds, axis=1) + elif args.output_mode == "regression": + preds = np.squeeze(preds) + result = compute_metrics(eval_task, preds, out_label_ids) + results.update(result) + + output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") + with open(output_eval_file, "w") as writer: + logger.info("***** Eval results {} *****".format(prefix)) + for key in sorted(result.keys()): + logger.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + return results + + +def load_and_cache_examples(args, task, tokenizer, evaluate=False): + if args.local_rank not in [-1, 0] and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + processor = processors[task](language=args.language, train_language=args.train_language) + output_mode = output_modes[task] + # Load data features from cache or dataset file + cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( + 'dev' if evaluate else 'train', + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length), + str(task), + str(args.train_language if (not evaluate and args.train_language is not None) else args.language))) + if os.path.exists(cached_features_file) and not args.overwrite_cache: + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", args.data_dir) + label_list = processor.get_labels() + examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + features = convert_examples_to_features(examples, + tokenizer, + label_list=label_list, + max_length=args.max_seq_length, + output_mode=output_mode, + pad_on_left=False, + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=0, + ) + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + + if args.local_rank == 0 and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + if output_mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) + else: + raise ValueError(f'No other `output_mode` for XNLI.') + + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) + return dataset + + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + parser.add_argument("--model_type", default=None, type=str, required=True, + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument("--model_name_or_path", default=None, type=str, required=True, + help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + parser.add_argument("--language", default=None, type=str, required=True, + help="Evaluation language. Also train language if `train_language` is set to None.") + parser.add_argument("--train_language", default=None, type=str, + help="Train language if is different of the evaluation language.") + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + + ## Other parameters + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--cache_dir", default="", type=str, + help="Where do you want to store the pre-trained models downloaded from s3") + parser.add_argument("--max_seq_length", default=128, type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.") + parser.add_argument("--do_train", action='store_true', + help="Whether to run training.") + parser.add_argument("--do_eval", action='store_true', + help="Whether to run eval on the dev set.") + parser.add_argument("--evaluate_during_training", action='store_true', + help="Rul evaluation during training at each logging step.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=3.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_steps", default=0, type=int, + help="Linear warmup over warmup_steps.") + + parser.add_argument('--logging_steps', type=int, default=50, + help="Log every X updates steps.") + parser.add_argument('--save_steps', type=int, default=50, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Avoid using CUDA when available") + parser.add_argument('--overwrite_output_dir', action='store_true', + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument('--tpu', action='store_true', + help="Whether to run on the TPU defined in the environment variables") + parser.add_argument('--tpu_ip_address', type=str, default='', + help="TPU IP address if none are set in the environment variables") + parser.add_argument('--tpu_name', type=str, default='', + help="TPU name if none are set in the environment variables") + parser.add_argument('--xrt_tpu_config', type=str, default='', + help="XRT TPU config if none are set in the environment variables") + + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument("--local_rank", type=int, default=-1, + help="For distributed training: local_rank") + parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") + args = parser.parse_args() + + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.device = device + + if args.tpu: + if args.tpu_ip_address: + os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address + if args.tpu_name: + os.environ["TPU_NAME"] = args.tpu_name + if args.xrt_tpu_config: + os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config + + assert "TPU_IP_ADDRESS" in os.environ + assert "TPU_NAME" in os.environ + assert "XRT_TPU_CONFIG" in os.environ + + import torch_xla + import torch_xla.core.xla_model as xm + args.device = xm.xla_device() + args.xla_model = xm + + # Setup logging + logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + + # Set seed + set_seed(args) + + # Prepare XNLI task + args.task_name = 'xnli' + if args.task_name not in processors: + raise ValueError("Task not found: %s" % (args.task_name)) + processor = processors[args.task_name](language=args.language, train_language=args.train_language) + args.output_mode = output_modes[args.task_name] + label_list = processor.get_labels() + num_labels = len(label_list) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) + model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + model.to(args.device) + + logger.info("Training/evaluation parameters %s", args) + + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False) + global_step, tr_loss = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + + # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir) + tokenizer.save_pretrained(args.output_dir) + + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + # Load a trained model and vocabulary that you have fine-tuned + model = model_class.from_pretrained(args.output_dir) + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + model.to(args.device) + + + # Evaluation + results = {} + if args.do_eval and args.local_rank in [-1, 0]: + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logger.info("Evaluate the following checkpoints: %s", checkpoints) + for checkpoint in checkpoints: + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else "" + + model = model_class.from_pretrained(checkpoint) + model.to(args.device) + result = evaluate(args, model, tokenizer, prefix=prefix) + result = dict((k + '_{}'.format(global_step), v) for k, v in result.items()) + results.update(result) + + return results + + +if __name__ == "__main__": + main() diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py new file mode 100644 index 0000000000..f0238f4664 --- /dev/null +++ b/examples/utils_xnli.py @@ -0,0 +1,93 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XNLI utils (dataset loading and evaluation) """ + +from __future__ import absolute_import, division, print_function + +import logging +import os + +from transformers.data.processors import DataProcessor, InputExample +from transformers.data.metrics import simple_accuracy + +logger = logging.getLogger(__name__) + +class XnliProcessor(DataProcessor): + """Processor for the XNLI dataset. + Adapted from https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/run_classifier.py#L207""" + + def __init__(self, language, train_language = None): + self.language = language + self.train_language = train_language + + def get_train_examples(self, data_dir): + """See base class.""" + lg = self.language if self.train_language is None else self.train_language + lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % ('train', i) + text_a = line[0] + text_b = line[1] + label = "contradiction" if line[2] == "contradictory" else line[2] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + language = line[0] + if language != self.language: + continue + guid = "%s-%s" % ('dev', i) + text_a = line[6] + text_b = line[7] + label = line[1] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + +def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise ValueError(f'{task_name} is not a supported task.') + +xnli_processors = { + "xnli": XnliProcessor, +} + +xnli_output_modes = { + "xnli": "classification", +} + +xnli_tasks_num_labels = { + "xnli": 3, +} From d52e98ff9af4509bb803641f0c0d81d67ce73cc3 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 11:51:15 -0400 Subject: [PATCH 76/91] add xnli examples/README.md --- examples/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/examples/README.md b/examples/README.md index e109a12171..6f8d6bd26e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -21,6 +21,7 @@ pip install [--editable] . | [SQuAD](#squad) | Using BERT/RoBERTa/XLNet/XLM for question answering, examples with distributed training. | | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training. | +| [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. | | [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. | ## TensorFlow 2.0 Bert models on GLUE @@ -600,3 +601,42 @@ python run_summarization_finetuning.py \ --do_train \ --data_path=$DATA_PATH \ ``` + +## XNLI + +Based on the script [`run_xnli.py`](TODO). + +[XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +#### Fine-tuning on XNLI + +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min +on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a +`$XNLI_DIR` directory. + +* [XNLI 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip) +* [XNLI-MT 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-MT-1.0.zip) + +```bash +export XNLI_DIR=/path/to/XNLI + +python run_xnli.py \ + --model_type bert \ + --model_name_or_path bert-base-multilingual-cased \ + --language en \ + --train_language en \ + --do_train \ + --do_eval \ + --data_dir $SQUAD_DIR \ + --per_gpu_train_batch_size 32 \ + --learning_rate 5e-5 \ + --num_train_epochs 2.0 \ + --max_seq_length 128 \ + --output_dir /tmp/debug_xnli/ +``` + +Training with the previously defined hyper-parameters yields the following results: + +```bash +TODO +``` From c4336ecbbdbb8bddcbfb31e0fcc4d382b430a9a5 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 12:04:20 -0400 Subject: [PATCH 77/91] xnli - output_mode consistency --- examples/run_xnli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index ee37296832..a9b2e46c13 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -247,8 +247,8 @@ def evaluate(args, model, tokenizer, prefix=""): eval_loss = eval_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) - elif args.output_mode == "regression": - preds = np.squeeze(preds) + else: + raise ValueError(f'No other `output_mode` for XNLI.') result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) From 84a0b522cf1b3f69bfbc92eb2309ba5ff652e521 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 18:53:45 -0400 Subject: [PATCH 78/91] mbert reproducibility results --- examples/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/README.md b/examples/README.md index 6f8d6bd26e..3e5fd03c45 100644 --- a/examples/README.md +++ b/examples/README.md @@ -604,13 +604,13 @@ python run_summarization_finetuning.py \ ## XNLI -Based on the script [`run_xnli.py`](TODO). +Based on the script [`run_xnli.py`](https://github.com/huggingface/transformers/blob/master/examples/run_xnli.py). [XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). #### Fine-tuning on XNLI -This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in 106 mins on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a `$XNLI_DIR` directory. @@ -623,20 +623,21 @@ export XNLI_DIR=/path/to/XNLI python run_xnli.py \ --model_type bert \ --model_name_or_path bert-base-multilingual-cased \ - --language en \ + --language es \ --train_language en \ --do_train \ --do_eval \ - --data_dir $SQUAD_DIR \ + --data_dir $XNLI_DIR \ --per_gpu_train_batch_size 32 \ --learning_rate 5e-5 \ --num_train_epochs 2.0 \ --max_seq_length 128 \ - --output_dir /tmp/debug_xnli/ + --output_dir /tmp/debug_xnli/ \ + --save_steps -1 ``` -Training with the previously defined hyper-parameters yields the following results: +Training with the previously defined hyper-parameters yields the following results on the dev set: ```bash -TODO +acc = 0.738152610441767 ``` From cb7b77a8a2ee52812c4358817a6a586f19687cda Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 30 Oct 2019 18:13:52 -0400 Subject: [PATCH 79/91] fix some typos --- transformers/tokenization_xlm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index 01f8721d98..ba994dc356 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tokenization classes for OpenAI GPT.""" +"""Tokenization classes for XLM.""" from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -758,9 +758,9 @@ class XLMTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: + A XLM sequence has the following format: single sequence: X - pair of sequences: A B + pair of sequences: A B """ if token_ids_1 is None: return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] From 289cf4d2b7cf090942e2e7dc4e6134a81095adf6 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:10:02 -0500 Subject: [PATCH 80/91] change default for XNLI: dev --> test --- examples/run_xnli.py | 6 +++--- examples/utils_xnli.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index a9b2e46c13..7fbbf7d152 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -270,7 +270,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): output_mode = output_modes[task] # Load data features from cache or dataset file cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( - 'dev' if evaluate else 'train', + 'test' if evaluate else 'train', list(filter(None, args.model_name_or_path.split('/'))).pop(), str(args.max_seq_length), str(task), @@ -281,7 +281,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): else: logger.info("Creating features from dataset file at %s", args.data_dir) label_list = processor.get_labels() - examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + examples = processor.get_test_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, tokenizer, label_list=label_list, @@ -341,7 +341,7 @@ def main(): parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', - help="Whether to run eval on the dev set.") + help="Whether to run eval on the test set.") parser.add_argument("--evaluate_during_training", action='store_true', help="Rul evaluation during training at each logging step.") parser.add_argument("--do_lower_case", action='store_true', diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py index f0238f4664..482e79a81f 100644 --- a/examples/utils_xnli.py +++ b/examples/utils_xnli.py @@ -50,9 +50,9 @@ class XnliProcessor(DataProcessor): InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples - def get_dev_examples(self, data_dir): + def get_test_examples(self, data_dir): """See base class.""" - lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv")) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv")) examples = [] for (i, line) in enumerate(lines): if i == 0: @@ -60,7 +60,7 @@ class XnliProcessor(DataProcessor): language = line[0] if language != self.language: continue - guid = "%s-%s" % ('dev', i) + guid = "%s-%s" % ('test', i) text_a = line[6] text_b = line[7] label = line[1] From d5910b312fbe02a4b08097646e46399dac179084 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:21:25 -0500 Subject: [PATCH 81/91] move xnli processor (and utils) to transformers/data/processors --- examples/run_xnli.py | 6 +++--- transformers/__init__.py | 3 ++- transformers/data/__init__.py | 1 + transformers/data/processors/__init__.py | 2 +- .../utils_xnli.py => transformers/data/processors/xnli.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) rename examples/utils_xnli.py => transformers/data/processors/xnli.py (97%) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 7fbbf7d152..0c1743e6c4 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -44,9 +44,9 @@ from transformers import (WEIGHTS_NAME, from transformers import AdamW, WarmupLinearSchedule -from utils_xnli import xnli_compute_metrics as compute_metrics -from utils_xnli import xnli_output_modes as output_modes -from utils_xnli import xnli_processors as processors +from transformers import xnli_compute_metrics as compute_metrics +from transformers import xnli_output_modes as output_modes +from transformers import xnli_processors as processors from transformers import glue_convert_examples_to_features as convert_examples_to_features diff --git a/transformers/__init__.py b/transformers/__init__.py index dff2479d4b..a133425a9c 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -25,7 +25,8 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, glue_output_modes, glue_convert_examples_to_features, - glue_processors, glue_tasks_num_labels) + glue_processors, glue_tasks_num_labels, + xnli_output_modes, xnli_processors, xnli_tasks_num_labels) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index e910d6da2e..46615608a4 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,5 +1,6 @@ from .processors import InputExample, InputFeatures, DataProcessor from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_labels from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index af38c54beb..1c41553ba4 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,3 +1,3 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features - +from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels diff --git a/examples/utils_xnli.py b/transformers/data/processors/xnli.py similarity index 97% rename from examples/utils_xnli.py rename to transformers/data/processors/xnli.py index 482e79a81f..96fac761a9 100644 --- a/examples/utils_xnli.py +++ b/transformers/data/processors/xnli.py @@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function import logging import os -from transformers.data.processors import DataProcessor, InputExample +from .utils import DataProcessor, InputExample from transformers.data.metrics import simple_accuracy logger = logging.getLogger(__name__) From d75d49a51de28df48b39280b2f06df78d3fa04bf Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:32:20 -0500 Subject: [PATCH 82/91] add XnliProcessor to doc --- docs/source/main_classes/processors.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index a85c126956..a093e621ad 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -55,4 +55,27 @@ Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ An example using these processors is given in the -`run_glue.py `__ script. \ No newline at end of file +`run_glue.py `__ script. + + +XNLI +~~~~~~~~~~~~~~~~~~~~~ + +`The Cross-Lingual NLI Corpus (XNLI) `__ is a benchmark that evaluates +the quality of cross-lingual text representations. +XNLI is crowd-sourced dataset based on `MultiNLI `: pairs of text are labeled with textual entailment +annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +It was released together with the paper +`XNLI: Evaluating Cross-lingual Sentence Representations `__ + +This library hosts the processor to load the XNLI data: + - :class:`~transformers.data.processors.utils.XnliProcessor` + +Please note that since the gold labels are available on the test set, evaluation is performed on the test set. + +Example usage +^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example using these processors is given in the +`run_xnli.py `__ script. \ No newline at end of file From abd397e95483cc7486bed47fd271ab417e1a288e Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:57:24 -0500 Subject: [PATCH 83/91] uniformize w/ the cache_dir update --- examples/run_xnli.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 0c1743e6c4..e8457e542b 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -467,9 +467,17 @@ def main(): args.model_type = args.model_type.lower() config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] - config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) - tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) - model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, + num_labels=num_labels, + finetuning_task=args.task_name, + cache_dir=args.cache_dir if args.cache_dir else None) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, + do_lower_case=args.do_lower_case, + cache_dir=args.cache_dir if args.cache_dir else None) + model = model_class.from_pretrained(args.model_name_or_path, + from_tf=bool('.ckpt' in args.model_name_or_path), + config=config, + cache_dir=args.cache_dir if args.cache_dir else None) if args.local_rank == 0: torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab From 3e7656f7ac369de08a2ebac3cd1d6870e60605bc Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 11:58:53 -0500 Subject: [PATCH 84/91] update readme --- examples/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/README.md b/examples/README.md index 3e5fd03c45..960b218f11 100644 --- a/examples/README.md +++ b/examples/README.md @@ -623,7 +623,7 @@ export XNLI_DIR=/path/to/XNLI python run_xnli.py \ --model_type bert \ --model_name_or_path bert-base-multilingual-cased \ - --language es \ + --language de \ --train_language en \ --do_train \ --do_eval \ @@ -636,8 +636,8 @@ python run_xnli.py \ --save_steps -1 ``` -Training with the previously defined hyper-parameters yields the following results on the dev set: +Training with the previously defined hyper-parameters yields the following results on the **test** set: ```bash -acc = 0.738152610441767 +acc = 0.7093812375249501 ``` From 73fe2e7385af4c6062f366825af570d44cd22fd8 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:51:43 -0500 Subject: [PATCH 85/91] remove fstrings --- examples/run_xnli.py | 4 ++-- transformers/data/processors/xnli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index e8457e542b..952128f4ad 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -248,7 +248,7 @@ def evaluate(args, model, tokenizer, prefix=""): if args.output_mode == "classification": preds = np.argmax(preds, axis=1) else: - raise ValueError(f'No other `output_mode` for XNLI.') + raise ValueError('No other `output_mode` for XNLI.') result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) @@ -305,7 +305,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): if output_mode == "classification": all_labels = torch.tensor([f.label for f in features], dtype=torch.long) else: - raise ValueError(f'No other `output_mode` for XNLI.') + raise ValueError('No other `output_mode` for XNLI.') dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index 96fac761a9..a4807dd901 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -36,7 +36,7 @@ class XnliProcessor(DataProcessor): def get_train_examples(self, data_dir): """See base class.""" lg = self.language if self.train_language is None else self.train_language - lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg))) examples = [] for (i, line) in enumerate(lines): if i == 0: @@ -78,7 +78,7 @@ def xnli_compute_metrics(task_name, preds, labels): if task_name == "xnli": return {"acc": simple_accuracy(preds, labels)} else: - raise ValueError(f'{task_name} is not a supported task.') + raise ValueError('{} is not a supported task.'.format(task_name)) xnli_processors = { "xnli": XnliProcessor, From bcd8dc6b48a335b899f00b59f016f740c5230d41 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:53:08 -0500 Subject: [PATCH 86/91] move xnli_compute_metrics to data/metrics --- transformers/__init__.py | 2 +- transformers/data/__init__.py | 2 +- transformers/data/metrics/__init__.py | 8 ++++++++ transformers/data/processors/xnli.py | 7 ------- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index a133425a9c..b29ad38e73 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -29,7 +29,7 @@ from .data import (is_sklearn_available, xnli_output_modes, xnli_processors, xnli_tasks_num_labels) if is_sklearn_available(): - from .data import glue_compute_metrics + from .data import glue_compute_metrics, xnli_compute_metrics # Tokenizers from .tokenization_utils import (PreTrainedTokenizer) diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index 46615608a4..b811a35807 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -4,4 +4,4 @@ from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_label from .metrics import is_sklearn_available if is_sklearn_available(): - from .metrics import glue_compute_metrics + from .metrics import glue_compute_metrics, xnli_compute_metrics diff --git a/transformers/data/metrics/__init__.py b/transformers/data/metrics/__init__.py index c9ebaac38d..5a46eb05d3 100644 --- a/transformers/data/metrics/__init__.py +++ b/transformers/data/metrics/__init__.py @@ -81,3 +81,11 @@ if _has_sklearn: return {"acc": simple_accuracy(preds, labels)} else: raise KeyError(task_name) + + + def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index a4807dd901..ce582f31a6 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -73,13 +73,6 @@ class XnliProcessor(DataProcessor): """See base class.""" return ["contradiction", "entailment", "neutral"] -def xnli_compute_metrics(task_name, preds, labels): - assert len(preds) == len(labels) - if task_name == "xnli": - return {"acc": simple_accuracy(preds, labels)} - else: - raise ValueError('{} is not a supported task.'.format(task_name)) - xnli_processors = { "xnli": XnliProcessor, } From d47402263964e30ee17cbc06811622bf2df50d6d Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:56:03 -0500 Subject: [PATCH 87/91] cleaning simple_accuracy since not used anymore --- transformers/data/processors/xnli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index ce582f31a6..efbe3762f6 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -21,7 +21,6 @@ import logging import os from .utils import DataProcessor, InputExample -from transformers.data.metrics import simple_accuracy logger = logging.getLogger(__name__) From 07ab8d7af6041f0d3562badcb3d6f173062329a1 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 17:33:14 -0500 Subject: [PATCH 88/91] fix bug --- transformers/data/processors/xnli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index efbe3762f6..958bdf62f9 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -35,7 +35,7 @@ class XnliProcessor(DataProcessor): def get_train_examples(self, data_dir): """See base class.""" lg = self.language if self.train_language is None else self.train_language - lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg))) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{}.tsv".format(lg))) examples = [] for (i, line) in enumerate(lines): if i == 0: From d5478b939d64db58972e46b7218c765c918b76ac Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 25 Nov 2019 19:40:48 +0000 Subject: [PATCH 89/91] add distilbert + update run_xnli wrt run_glue --- examples/run_xnli.py | 51 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 952128f4ad..a3bc0d4604 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Finetuning multi-lingual models on XNLI (Bert, XLM). +""" Finetuning multi-lingual models on XNLI (Bert, DistilBERT, XLM). Adapted from `examples/run_glue.py`""" from __future__ import absolute_import, division, print_function @@ -42,7 +42,7 @@ from transformers import (WEIGHTS_NAME, XLMConfig, XLMForSequenceClassification, XLMTokenizer, DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from transformers import xnli_compute_metrics as compute_metrics from transformers import xnli_output_modes as output_modes @@ -52,12 +52,12 @@ from transformers import glue_convert_examples_to_features as convert_examples_t logger = logging.getLogger(__name__) -ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ()) +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, DistilBertConfig, XLMConfig)), ()) MODEL_CLASSES = { 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), - # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) } @@ -91,7 +91,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp @@ -149,7 +149,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -180,11 +180,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -214,6 +209,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu eval + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) @@ -383,15 +382,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -425,23 +415,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -495,7 +468,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) @@ -512,7 +485,7 @@ def main(): # Load a trained model and vocabulary that you have fine-tuned model = model_class.from_pretrained(args.output_dir) - tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + tokenizer = tokenizer_class.from_pretrained(args.output_dir) model.to(args.device) From 10bd1ddb39235b2f58594e48867595e7d38cd619 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 25 Nov 2019 19:41:00 +0000 Subject: [PATCH 90/91] soft launch distilbert multilingual --- transformers/configuration_distilbert.py | 3 ++- transformers/modeling_distilbert.py | 3 ++- transformers/tokenization_distilbert.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index 2a8a149acf..f929a9bc39 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -27,7 +27,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-config.json", } diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index d30f493c69..62c623ff6c 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -42,7 +42,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-pytorch_model.bin", } diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index dfa02926d8..832f0c3d0b 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -33,12 +33,14 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'distilbert-base-uncased': 512, 'distilbert-base-uncased-distilled-squad': 512, + 'distilbert-base-multilingual-cased': 512, } From 3c28a2daac43386dbc63b0bd014a966f22888850 Mon Sep 17 00:00:00 2001 From: Yao Lu <95luyao@gmail.com> Date: Wed, 27 Nov 2019 11:45:22 -0500 Subject: [PATCH 91/91] add add_special_tokens=True for input examples --- transformers/modeling_bert.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 81d92d8f1b..34bb8f89ba 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -597,7 +597,7 @@ class BertModel(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertModel.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -760,7 +760,7 @@ class BertForPreTraining(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -836,7 +836,7 @@ class BertForMaskedLM(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -919,7 +919,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -984,7 +984,7 @@ class BertForSequenceClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -1060,7 +1060,7 @@ class BertForMultipleChoice(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMultipleChoice.from_pretrained('bert-base-uncased') choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices labels = torch.tensor(1).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, classification_scores = outputs[:2] @@ -1134,7 +1134,7 @@ class BertForTokenClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, scores = outputs[:2]