From a9f24a16bc2965d2990b90127ed4b5a1f47344b9 Mon Sep 17 00:00:00 2001
From: mataney <mataneyal1@gmail.com>
Date: Wed, 25 Sep 2019 15:53:29 +0300
Subject: [PATCH 001/269] [FIX] fix run_generation.py to work with batch_size >
 1

---
 examples/run_generation.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/examples/run_generation.py b/examples/run_generation.py
index a2a8f29103..935e578441 100644
--- a/examples/run_generation.py
+++ b/examples/run_generation.py
@@ -81,7 +81,6 @@ def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')
                 Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)
         From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317
     """
-    assert logits.dim() == 1  # batch size 1 for now - could be updated for more but the code would be less clear
     top_k = min(top_k, logits.size(-1))  # Safety check
     if top_k > 0:
         # Remove all tokens with a probability less than the last token of the top-k
@@ -98,7 +97,8 @@ def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')
         sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
         sorted_indices_to_remove[..., 0] = 0
 
-        indices_to_remove = sorted_indices[sorted_indices_to_remove]
+        # scatter sorted tensors to original indexing
+        indices_to_remove = sorted_indices_to_remove.scatter(dim=1, index=sorted_indices, src=sorted_indices_to_remove)
         logits[indices_to_remove] = filter_value
     return logits
 
@@ -122,10 +122,10 @@ def sample_sequence(model, length, context, num_samples=1, temperature=1, top_k=
                 inputs = {'input_ids': input_ids, 'perm_mask': perm_mask, 'target_mapping': target_mapping}
 
             outputs = model(**inputs)  # Note: we could also use 'past' with GPT-2/Transfo-XL/XLNet (cached hidden-states)
-            next_token_logits = outputs[0][0, -1, :] / temperature
+            next_token_logits = outputs[0][:, -1, :] / temperature
             filtered_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)
             next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1), num_samples=1)
-            generated = torch.cat((generated, next_token.unsqueeze(0)), dim=1)
+            generated = torch.cat((generated, next_token), dim=1)
     return generated
 
 
@@ -139,6 +139,7 @@ def main():
     parser.add_argument("--padding_text", type=str, default="")
     parser.add_argument("--length", type=int, default=20)
     parser.add_argument("--temperature", type=float, default=1.0)
+    parser.add_argument("--num_samples", type=int, default=1)
     parser.add_argument("--top_k", type=int, default=0)
     parser.add_argument("--top_p", type=float, default=0.9)
     parser.add_argument("--no_cuda", action='store_true',
@@ -176,6 +177,7 @@ def main():
         out = sample_sequence(
             model=model,
             context=context_tokens,
+            num_samples=args.num_samples,
             length=args.length,
             temperature=args.temperature,
             top_k=args.top_k,
@@ -183,9 +185,10 @@ def main():
             device=args.device,
             is_xlnet=bool(args.model_type == "xlnet"),
         )
-        out = out[0, len(context_tokens):].tolist()
-        text = tokenizer.decode(out, clean_up_tokenization_spaces=True)
-        print(text)
+        out = out[:, len(context_tokens):].tolist()
+        for o in out:
+            text = tokenizer.decode(o, clean_up_tokenization_spaces=True)
+            print(text)
         if args.prompt:
             break
     return text

From 4446c02b8a277325d7b145554b301919121902c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Mon, 7 Oct 2019 12:04:05 +0200
Subject: [PATCH 002/269] add wireframe for seq2seq model

---
 transformers/modeling_seq2seq.py            | 37 +++++++++++++++++++++
 transformers/tests/modeling_seq2seq_test.py | 23 +++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 transformers/modeling_seq2seq.py
 create mode 100644 transformers/tests/modeling_seq2seq_test.py

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
new file mode 100644
index 0000000000..990f35ffed
--- /dev/null
+++ b/transformers/modeling_seq2seq.py
@@ -0,0 +1,37 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Conditional generation class. """
+
+
+class Seq2SeqModel(object):
+    def __init__(self):
+        # need to use from_pretrained to initialize
+        raise NotImplementedError
+
+    @classmethod
+    def from_pretrained(cls, encoder, decoder):
+        # Here we should call AutoModel to initialize the models depending
+        # on the pretrained models taken as an input.
+        # For a first iteration we only work with Bert.
+        raise NotImplementedError
+
+    def __call__(self):
+        # allows to call an instance of the class
+        # model = Seq2Seq(encode='bert', decoder='bert')
+        raise NotImplementedError
+
+    def process(self):
+        # alternative API to __call__ it is more explicit.
+        raise NotImplementedError
diff --git a/transformers/tests/modeling_seq2seq_test.py b/transformers/tests/modeling_seq2seq_test.py
new file mode 100644
index 0000000000..1866dc10af
--- /dev/null
+++ b/transformers/tests/modeling_seq2seq_test.py
@@ -0,0 +1,23 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+
+class Seq2SeqTest(unittest.TestCase):
+    raise NotImplementedError
+
+
+def __main__():
+    unittest.main()

From 386e86e22288b08bf8bcc3cff4027c46ba866d91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Mon, 7 Oct 2019 13:00:06 +0200
Subject: [PATCH 003/269] raise exception when class initialized with __init__

---
 transformers/modeling_seq2seq.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 990f35ffed..b14622e50f 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -17,11 +17,13 @@
 
 class Seq2SeqModel(object):
     def __init__(self):
-        # need to use from_pretrained to initialize
-        raise NotImplementedError
+        raise EnvironmentError(
+            """Seq2Seq is designed to be instantiated using the
+        `Seq2Seq.from_pretrained(encoder_name_or_path, decoder_name_or_path)` method."""
+        )
 
     @classmethod
-    def from_pretrained(cls, encoder, decoder):
+    def from_pretrained(cls, encoder_name, decoder_name):
         # Here we should call AutoModel to initialize the models depending
         # on the pretrained models taken as an input.
         # For a first iteration we only work with Bert.

From 0053c0e052d0c4c0eb63faaaf61b8140122f8444 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Mon, 7 Oct 2019 16:29:15 +0200
Subject: [PATCH 004/269] do some (light) housekeeping

Several packages were imported but never used, indentation and line
spaces did not follow PEP8.
---
 transformers/modeling_bert.py | 59 ++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index fc448fa366..3187d1ca50 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -17,12 +17,10 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import json
 import logging
 import math
 import os
 import sys
-from io import open
 
 import torch
 from torch import nn
@@ -50,6 +48,7 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
     'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin",
 }
 
+
 def load_tf_weights_in_bert(model, config, tf_checkpoint_path):
     """ Load tf checkpoints in a pytorch model.
     """
@@ -125,12 +124,14 @@ def gelu(x):
     """
     return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
 
+
 def gelu_new(x):
     """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
         Also see https://arxiv.org/abs/1606.08415
     """
     return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
 
+
 def swish(x):
     return x * torch.sigmoid(x)
 
@@ -140,6 +141,7 @@ ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_
 
 BertLayerNorm = torch.nn.LayerNorm
 
+
 class BertEmbeddings(nn.Module):
     """Construct the embeddings from word, position and token_type embeddings.
     """
@@ -482,7 +484,7 @@ BERT_START_DOCSTRING = r"""    The BERT model was proposed in
         https://pytorch.org/docs/stable/nn.html#module
 
     Parameters:
-        config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. 
+        config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model.
             Initializing with a config file does not load the weights associated with the model, only the configuration.
             Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
 """
@@ -496,13 +498,13 @@ BERT_INPUTS_DOCSTRING = r"""
             (a) For sequence pairs:
 
                 ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]``
-                
+
                 ``token_type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1``
 
             (b) For single sequences:
 
                 ``tokens:         [CLS] the dog is hairy . [SEP]``
-                
+
                 ``token_type_ids:   0   0   0   0  0     0   0``
 
             Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on
@@ -601,7 +603,7 @@ class BertModel(BertPreTrainedModel):
         # positions we want to attend and -10000.0 for masked positions.
         # Since we are adding it to the raw scores before the softmax, this is
         # effectively the same as removing these entirely.
-        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility
+        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
         extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
 
         # Prepare head mask if needed
@@ -615,7 +617,7 @@ class BertModel(BertPreTrainedModel):
                 head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1)
             elif head_mask.dim() == 2:
                 head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
-            head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility
+            head_mask = head_mask.to(dtype=next(self.parameters()).dtype)  # switch to fload if need + fp16 compatibility
         else:
             head_mask = [None] * self.config.num_hidden_layers
 
@@ -631,8 +633,9 @@ class BertModel(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model with two heads on top as done during the pre-training:
-    a `masked language modeling` head and a `next sentence prediction (classification)` head. """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                       a `masked language modeling` head and a `next sentence prediction (classification)` head. """,
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForPreTraining(BertPreTrainedModel):
     r"""
         **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
@@ -692,7 +695,7 @@ class BertForPreTraining(BertPreTrainedModel):
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
+                            position_ids=position_ids,
                             head_mask=head_mask)
 
         sequence_output, pooled_output = outputs[:2]
@@ -711,7 +714,8 @@ class BertForPreTraining(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model with a `language modeling` head on top. """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForMaskedLM(BertPreTrainedModel):
     r"""
         **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
@@ -764,7 +768,7 @@ class BertForMaskedLM(BertPreTrainedModel):
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
+                            position_ids=position_ids,
                             head_mask=head_mask)
 
         sequence_output = outputs[0]
@@ -780,7 +784,8 @@ class BertForMaskedLM(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForNextSentencePrediction(BertPreTrainedModel):
     r"""
         **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
@@ -825,7 +830,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel):
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
+                            position_ids=position_ids,
                             head_mask=head_mask)
 
         pooled_output = outputs[1]
@@ -842,8 +847,9 @@ class BertForNextSentencePrediction(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of
-    the pooled output) e.g. for GLUE tasks. """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      the pooled output) e.g. for GLUE tasks. """,
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForSequenceClassification(BertPreTrainedModel):
     r"""
         **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
@@ -891,7 +897,7 @@ class BertForSequenceClassification(BertPreTrainedModel):
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
+                            position_ids=position_ids,
                             head_mask=head_mask)
 
         pooled_output = outputs[1]
@@ -915,8 +921,9 @@ class BertForSequenceClassification(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of
-    the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """,
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForMultipleChoice(BertPreTrainedModel):
     r"""
         **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
@@ -990,8 +997,9 @@ class BertForMultipleChoice(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model with a token classification head on top (a linear layer on top of
-    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForTokenClassification(BertPreTrainedModel):
     r"""
         **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
@@ -1037,7 +1045,7 @@ class BertForTokenClassification(BertPreTrainedModel):
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
+                            position_ids=position_ids,
                             head_mask=head_mask)
 
         sequence_output = outputs[0]
@@ -1062,8 +1070,9 @@ class BertForTokenClassification(BertPreTrainedModel):
 
 
 @add_start_docstrings("""Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of
-    the hidden-states output to compute `span start logits` and `span end logits`). """,
-    BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      the hidden-states output to compute `span start logits` and `span end logits`). """,
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
 class BertForQuestionAnswering(BertPreTrainedModel):
     r"""
         **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
@@ -1116,7 +1125,7 @@ class BertForQuestionAnswering(BertPreTrainedModel):
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
+                            position_ids=position_ids,
                             head_mask=head_mask)
 
         sequence_output = outputs[0]

From dda1adad6de0874e337ec04e52c4c291c0abfb59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Mon, 7 Oct 2019 16:31:46 +0200
Subject: [PATCH 005/269] rename BertLayer to BertEncoderLayer

---
 transformers/modeling_bert.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 3187d1ca50..20b49c592f 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -315,9 +315,9 @@ class BertOutput(nn.Module):
         return hidden_states
 
 
-class BertLayer(nn.Module):
+class BertEncoderLayer(nn.Module):
     def __init__(self, config):
-        super(BertLayer, self).__init__()
+        super(BertEncoderLayer, self).__init__()
         self.attention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
@@ -336,7 +336,7 @@ class BertEncoder(nn.Module):
         super(BertEncoder, self).__init__()
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
-        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
+        self.layer = nn.ModuleList([BertEncoderLayer(config) for _ in range(config.num_hidden_layers)])
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
         all_hidden_states = ()

From 31adbb247c8c3ec248e30f89a3e4278622915ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Mon, 7 Oct 2019 16:43:21 +0200
Subject: [PATCH 006/269] add class wireframes for Bert decoder

---
 transformers/modeling_bert.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 20b49c592f..f2e2dba589 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -331,6 +331,14 @@ class BertEncoderLayer(nn.Module):
         return outputs
 
 
+class BertDecoderLayer(nn.Module):
+    def __init__(self, config):
+        raise NotImplementedError
+
+    def forward(self, hidden_state, encoder_output):
+        raise NotImplementedError
+
+
 class BertEncoder(nn.Module):
     def __init__(self, config):
         super(BertEncoder, self).__init__()
@@ -363,6 +371,14 @@ class BertEncoder(nn.Module):
         return outputs  # last-layer hidden state, (all hidden states), (all attentions)
 
 
+class BertDecoder(nn.Module):
+    def __init__(self, config):
+        raise NotImplementedError
+
+    def forward(self, encoder_output):
+        raise NotImplementedError
+
+
 class BertPooler(nn.Module):
     def __init__(self, config):
         super(BertPooler, self).__init__()

From a0dcefa382a541d0fecd634d6d0c3f97cd221faf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Mon, 7 Oct 2019 17:53:58 +0200
Subject: [PATCH 007/269] generalize BertSelfAttention to take separate query,
 key, value

There is currently no way to specify the quey, key and value separately
in the Attention module. However, the decoder's "encoder-decoder
attention" layers take the decoder's last output as a query, the
encoder's states as key and value. We thus modify the existing code so
query, key and value can be added separately.

This obviously poses some naming conventions; `BertSelfAttention` is not
a self-attention module anymore. The way the residual is forwarded is
now awkard, etc. We will need to do some refacto once the decoder is
fully implemented.
---
 transformers/modeling_bert.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index f2e2dba589..8a2624f8f0 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -198,10 +198,10 @@ class BertSelfAttention(nn.Module):
         x = x.view(*new_x_shape)
         return x.permute(0, 2, 1, 3)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        mixed_query_layer = self.query(hidden_states)
-        mixed_key_layer = self.key(hidden_states)
-        mixed_value_layer = self.value(hidden_states)
+    def forward(self, query, key, value, attention_mask=None, head_mask=None):
+        mixed_query_layer = self.query(query)
+        mixed_key_layer = self.key(key)
+        mixed_value_layer = self.value(value)
 
         query_layer = self.transpose_for_scores(mixed_query_layer)
         key_layer = self.transpose_for_scores(mixed_key_layer)
@@ -279,9 +279,12 @@ class BertAttention(nn.Module):
         self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def forward(self, input_tensor, attention_mask=None, head_mask=None):
-        self_outputs = self.self(input_tensor, attention_mask, head_mask)
-        attention_output = self.output(self_outputs[0], input_tensor)
+    def forward(self, query_tensor, key_tensor, value_tensor, attention_mask=None, head_mask=None):
+        self_outputs = self.self(query_tensor, key_tensor, value_tensor, attention_mask, head_mask)
+        # in encoder-decoder attention we use the output of the previous decoder stage as the query
+        # in the Multi-Head Attention. We thus pass query_tensor as the residual in BertOutput.
+        # This shows the limits of the current code architecture, which may benefit from some refactoring.
+        attention_output = self.output(self_outputs[0], query_tensor)
         outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
         return outputs
 
@@ -323,7 +326,11 @@ class BertEncoderLayer(nn.Module):
         self.output = BertOutput(config)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
+        attention_outputs = self.attention(query_tensor=hidden_states,
+                                           key_tensor=hidden_states,
+                                           value_tensor=hidden_states,
+                                           attention_mask=attention_mask,
+                                           head_mask=head_mask)
         attention_output = attention_outputs[0]
         intermediate_output = self.intermediate(attention_output)
         layer_output = self.output(intermediate_output, attention_output)
@@ -333,6 +340,7 @@ class BertEncoderLayer(nn.Module):
 
 class BertDecoderLayer(nn.Module):
     def __init__(self, config):
+        super(BertDecoderLayer, self).__init__()
         raise NotImplementedError
 
     def forward(self, hidden_state, encoder_output):

From cd6a59d5c1cb9c7905675fc82ce50df5e2bdf3f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 10:11:02 +0200
Subject: [PATCH 008/269] add a decoder layer for Bert

---
 transformers/modeling_bert.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 8a2624f8f0..4011da18b4 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -341,10 +341,28 @@ class BertEncoderLayer(nn.Module):
 class BertDecoderLayer(nn.Module):
     def __init__(self, config):
         super(BertDecoderLayer, self).__init__()
-        raise NotImplementedError
+        self.self_attention = BertAttention(config)
+        self.attention = BertAttention(config)
+        self.intermediate = BertIntermediate(config)
+        self.output = BertOutput(config)
 
-    def forward(self, hidden_state, encoder_output):
-        raise NotImplementedError
+    def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
+        self_attention_outputs = self.self_attention(query_tensor=hidden_states,
+                                                     key_tensor=hidden_states,
+                                                     value_tensor=hidden_states,
+                                                     attention_mask=attention_mask,
+                                                     head_mask=head_mask)
+        self_attention_output = self_attention_outputs[0]
+        attention_outputs = self.attention(query_tensor=self_attention_output,
+                                           key_tensor=encoder_outputs,
+                                           value_tensor=encoder_outputs,
+                                           attention_mask=attention_mask,
+                                           head_mask=head_mask)
+        attention_output = attention_outputs[0]
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        outputs = (layer_output,) + attention_outputs[1:]
+        return outputs
 
 
 class BertEncoder(nn.Module):

From 15a2fc88a68741163cc9b798921e6b33ef32528a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 11:10:35 +0200
Subject: [PATCH 009/269] add General attention classes

The modifications that I introduced in a previous commit did break
Bert's internal API. I reverted these changes and added more general
classes to handle the encoder-decoder attention case.

There may be a more elegant way to deal with retro-compatibility (I am
not comfortable with the current state of the code), but I cannot see it
right now.
---
 transformers/modeling_bert.py | 158 +++++++++++++++++++++++++++++-----
 1 file changed, 136 insertions(+), 22 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 4011da18b4..a5e36eaed0 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -174,9 +174,9 @@ class BertEmbeddings(nn.Module):
         return embeddings
 
 
-class BertSelfAttention(nn.Module):
+class BertGeneralAttention(nn.Module):
     def __init__(self, config):
-        super(BertSelfAttention, self).__init__()
+        super(BertGeneralAttention, self).__init__()
         if config.hidden_size % config.num_attention_heads != 0:
             raise ValueError(
                 "The hidden size (%d) is not a multiple of the number of attention "
@@ -235,6 +235,67 @@ class BertSelfAttention(nn.Module):
         return outputs
 
 
+class BertSelfAttention(nn.Module):
+    def __init__(self, config):
+        super(BertSelfAttention, self).__init__()
+        if config.hidden_size % config.num_attention_heads != 0:
+            raise ValueError(
+                "The hidden size (%d) is not a multiple of the number of attention "
+                "heads (%d)" % (config.hidden_size, config.num_attention_heads))
+        self.output_attentions = config.output_attentions
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        mixed_query_layer = self.query(hidden_states)
+        mixed_key_layer = self.key(hidden_states)
+        mixed_value_layer = self.value(hidden_states)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+        key_layer = self.transpose_for_scores(mixed_key_layer)
+        value_layer = self.transpose_for_scores(mixed_value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,)
+        return outputs
+
+
 class BertSelfOutput(nn.Module):
     def __init__(self, config):
         super(BertSelfOutput, self).__init__()
@@ -279,12 +340,49 @@ class BertAttention(nn.Module):
         self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def forward(self, query_tensor, key_tensor, value_tensor, attention_mask=None, head_mask=None):
-        self_outputs = self.self(query_tensor, key_tensor, value_tensor, attention_mask, head_mask)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        self_outputs = self.self(hidden_states, attention_mask, head_mask)
+        attention_output = self.output(self_outputs[0], hidden_states)
+        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+class BertDecoderAttention(nn.Module):
+    def __init__(self, config):
+        super(BertAttention, self).__init__()
+        self.self = BertGeneralAttention(config)
+        self.output = BertSelfOutput(config)
+        self.pruned_heads = set()
+
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size)
+        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
+        for head in heads:
+            # Compute how many pruned heads are before the head and move the index accordingly
+            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
+            mask[head] = 0
+        mask = mask.view(-1).contiguous().eq(1)
+        index = torch.arange(len(mask))[mask].long()
+
+        # Prune linear layers
+        self.self.query = prune_linear_layer(self.self.query, index)
+        self.self.key = prune_linear_layer(self.self.key, index)
+        self.self.value = prune_linear_layer(self.self.value, index)
+        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
+        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
+
+    def forward(self, query, key, value, attention_mask=None, head_mask=None):
+        self_outputs = self.self(query, key, value, attention_mask, head_mask)
         # in encoder-decoder attention we use the output of the previous decoder stage as the query
         # in the Multi-Head Attention. We thus pass query_tensor as the residual in BertOutput.
         # This shows the limits of the current code architecture, which may benefit from some refactoring.
-        attention_output = self.output(self_outputs[0], query_tensor)
+        attention_output = self.output(self_outputs[0], query)
         outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
         return outputs
 
@@ -326,11 +424,7 @@ class BertEncoderLayer(nn.Module):
         self.output = BertOutput(config)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        attention_outputs = self.attention(query_tensor=hidden_states,
-                                           key_tensor=hidden_states,
-                                           value_tensor=hidden_states,
-                                           attention_mask=attention_mask,
-                                           head_mask=head_mask)
+        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
         attention_output = attention_outputs[0]
         intermediate_output = self.intermediate(attention_output)
         layer_output = self.output(intermediate_output, attention_output)
@@ -342,20 +436,16 @@ class BertDecoderLayer(nn.Module):
     def __init__(self, config):
         super(BertDecoderLayer, self).__init__()
         self.self_attention = BertAttention(config)
-        self.attention = BertAttention(config)
+        self.attention = BertDecoderAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
 
     def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
-        self_attention_outputs = self.self_attention(query_tensor=hidden_states,
-                                                     key_tensor=hidden_states,
-                                                     value_tensor=hidden_states,
-                                                     attention_mask=attention_mask,
-                                                     head_mask=head_mask)
+        self_attention_outputs = self.self_attention(hidden_states, attention_mask, head_mask)
         self_attention_output = self_attention_outputs[0]
-        attention_outputs = self.attention(query_tensor=self_attention_output,
-                                           key_tensor=encoder_outputs,
-                                           value_tensor=encoder_outputs,
+        attention_outputs = self.attention(query=self_attention_output,
+                                           key=encoder_outputs,
+                                           value=encoder_outputs,
                                            attention_mask=attention_mask,
                                            head_mask=head_mask)
         attention_output = attention_outputs[0]
@@ -399,10 +489,34 @@ class BertEncoder(nn.Module):
 
 class BertDecoder(nn.Module):
     def __init__(self, config):
-        raise NotImplementedError
+        super(BertDecoder, self).__init__()
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
+        self.layers = nn.ModuleList([BertEncoderLayer(config) for _ in range(config.num_hidden_layers)])
 
-    def forward(self, encoder_output):
-        raise NotImplementedError
+    def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
+        all_hidden_states = ()
+        all_attentions = ()
+        for i, layer_module in enumerate(self.layer):
+            if self.output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i])
+            if self.output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+            hidden_states = layer_outputs[0]
+
+        # Add last layer
+        if self.output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (all_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (all_attentions,)
+        return outputs  # last-layer hidden state, (all hidden states), (all attentions)
 
 
 class BertPooler(nn.Module):

From 75feacf172d5aa26314929c0f3bb54d9e845e00b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 11:39:04 +0200
Subject: [PATCH 010/269] add general structure for Bert2Bert class

---
 transformers/modeling_bert.py | 60 +++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index a5e36eaed0..1e228556b4 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -1310,3 +1310,63 @@ class BertForQuestionAnswering(BertPreTrainedModel):
             outputs = (total_loss,) + outputs
 
         return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)
+
+
+@add_start_docstrings("Bert encoder-decoder model",
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
+class Bert2Bert(BertPreTrainedModel):
+    r"""
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+
+    Examples::
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = Bert2Bert.from_pretrained('bert-base-uncased')
+        input = tokenizer.encode("Hello, how are you?")
+        outputs = model(input)
+        output_text = tokenize.decode(outputs[0])
+        print(output_text)
+    """
+
+    def __init__(self, config):
+        super(Bert2Bert, self).__init__(config)
+        self.embeddings = BertEmbeddings(config)
+        self.encoder = BertEncoder(config)
+        self.decoder = BertDecoder(config)
+
+    def forward(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+        if attention_mask is None:
+            attention_mask = torch.ones_like(inputs)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros_like(inputs)
+
+        # We create a 3D attention mask from a 2D tensor mask.
+        # Sizes are [batch_size, 1, 1, to_seq_length]
+        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+        # this attention mask is more simple than the triangular masking of causal attention
+        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+
+        embedding_output = self.embeddings(inputs, position_ids=position_ids, token_type_ids=token_type_ids)
+        encoder_outputs = self.encoder(embedding_output,
+                                       extended_attention_mask,
+                                       head_mask=head_mask)
+        decoder_outputs = self.decoder(embedding_output,
+                                       encoder_outputs[0],
+                                       extended_attention_mask,
+                                       head_mask=head_mask)
+        sequence_output = decoder_outputs[0]
+        pooled_output = self.pooler(sequence_output)
+
+        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)

From 070098309074e161ed1df35cf918013aeccba462 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 15:39:47 +0200
Subject: [PATCH 011/269] Add BertDecoderModel and Bert2Bert classes

I am not sure what happens when the class is initialized with the
pretrained weights.
---
 transformers/modeling_bert.py | 169 +++++++++++++++++++++++++++-------
 1 file changed, 134 insertions(+), 35 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 1e228556b4..9ce32d808e 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -788,6 +788,110 @@ class BertModel(BertPreTrainedModel):
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
 
 
+@add_start_docstrings("""A bare Bert decoder Model transformer outputting raw hidden-states without any specific head on top.
+                      The model follows the general transformer decoder architecture.""",
+                      BERT_START_DOCSTRING,
+                      BERT_INPUTS_DOCSTRING)
+class BertDecoderModel(BertPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
+            Sequence of hidden-states at the output of the last layer of the model.
+        **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)``
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during Bert pretraining. This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertDecoderModel.from_pretrained('bert-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
+    """
+    def __init__(self, config):
+        super(BertModel, self).__init__(config)
+
+        self.embeddings = BertEmbeddings(config)
+        self.decoder = BertDecoder(config)
+        self.pooler = BertPooler(config)
+
+        self.init_weights()
+
+    def _resize_token_embeddings(self, new_num_tokens):
+        old_embeddings = self.embeddings.word_embeddings
+        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
+        self.embeddings.word_embeddings = new_embeddings
+        return self.embeddings.word_embeddings
+
+    def _prune_heads(self, heads_to_prune):
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            See base class PreTrainedModel
+        """
+        for layer, heads in heads_to_prune.items():
+            self.encoder.layer[layer].attention.prune_heads(heads)
+
+    def forward(self, input_ids, encoder_outputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros_like(input_ids)
+
+        # We create a 3D attention mask from a 2D tensor mask.
+        # Sizes are [batch_size, 1, 1, to_seq_length]
+        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+        # this attention mask is more simple than the triangular masking of causal attention
+        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        if head_mask is not None:
+            if head_mask.dim() == 1:
+                head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
+                head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1)
+            elif head_mask.dim() == 2:
+                head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
+            head_mask = head_mask.to(dtype=next(self.parameters()).dtype)  # switch to fload if need + fp16 compatibility
+        else:
+            head_mask = [None] * self.config.num_hidden_layers
+
+        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        decoder_outputs = self.decoder(embedding_output,
+                                       encoder_outputs,
+                                       extended_attention_mask,
+                                       head_mask=head_mask)
+        sequence_output = decoder_outputs[0]
+        pooled_output = self.pooler(sequence_output)
+
+        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
+
+
 @add_start_docstrings("""Bert Model with two heads on top as done during the pre-training:
                        a `masked language modeling` head and a `next sentence prediction (classification)` head. """,
                       BERT_START_DOCSTRING,
@@ -1312,13 +1416,20 @@ class BertForQuestionAnswering(BertPreTrainedModel):
         return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)
 
 
-@add_start_docstrings("Bert encoder-decoder model",
+@add_start_docstrings("Bert encoder-decoder model for sequence generation.",
                       BERT_START_DOCSTRING,
                       BERT_INPUTS_DOCSTRING)
 class Bert2Bert(BertPreTrainedModel):
     r"""
 
     Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
 
     Examples::
 
@@ -1328,45 +1439,33 @@ class Bert2Bert(BertPreTrainedModel):
         outputs = model(input)
         output_text = tokenize.decode(outputs[0])
         print(output_text)
+
+    References::
+
+    [1] "Leveraging Pre-trained Checkpoints for Sequence Generation Tasks", S.Rothe, S.Narayan & A.Severyn (2019) ArXiV:1907.12461v1
+    [2] Tensor2Tensor library https://github.com/tensorflow/tensor2tensor
+
     """
 
     def __init__(self, config):
         super(Bert2Bert, self).__init__(config)
-        self.embeddings = BertEmbeddings(config)
-        self.encoder = BertEncoder(config)
-        self.decoder = BertDecoder(config)
+        self.encoder = BertModel(config)
+        self.decoder = BertDecoderModel(config)
 
-    def forward(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
-        if attention_mask is None:
-            attention_mask = torch.ones_like(inputs)
-        if token_type_ids is None:
-            token_type_ids = torch.zeros_like(inputs)
+        self.init_weights()
 
-        # We create a 3D attention mask from a 2D tensor mask.
-        # Sizes are [batch_size, 1, 1, to_seq_length]
-        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
-        # this attention mask is more simple than the triangular masking of causal attention
-        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
-        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
-
-        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
-        # masked positions, this operation will create a tensor which is 0.0 for
-        # positions we want to attend and -10000.0 for masked positions.
-        # Since we are adding it to the raw scores before the softmax, this is
-        # effectively the same as removing these entirely.
-        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
-        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
-
-        embedding_output = self.embeddings(inputs, position_ids=position_ids, token_type_ids=token_type_ids)
-        encoder_outputs = self.encoder(embedding_output,
-                                       extended_attention_mask,
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+        encoder_outputs = self.encoder(input_ids,
+                                       attention_mask=attention_mask,
+                                       token_type_ids=token_type_ids,
+                                       position_ids=position_ids,
                                        head_mask=head_mask)
-        decoder_outputs = self.decoder(embedding_output,
-                                       encoder_outputs[0],
-                                       extended_attention_mask,
-                                       head_mask=head_mask)
-        sequence_output = decoder_outputs[0]
-        pooled_output = self.pooler(sequence_output)
+        encoder_output = encoder_outputs[0]
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
-        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
+        decoder_input = torch.empty_like(input_ids).normal_(mean=0.0, std=self.config.initializer_range)
+        decoder_outputs = self.decoder(decoder_input,
+                                       encoder_output,
+                                       token_type_ids=token_type_ids,
+                                       position_ids=position_ids,
+                                       head_mask=head_mask)
+        return decoder_outputs

From 82628b0fc921e5e3f250bcad10f2b3c54111c17f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 15:57:25 +0200
Subject: [PATCH 012/269] add a placeholder test

---
 transformers/__init__.py                 |  2 +-
 transformers/tests/modeling_bert_test.py | 29 ++++++++++++++----------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 5248bc9f1b..bf302992b2 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -64,7 +64,7 @@ if is_torch_available():
                                 BertForMaskedLM, BertForNextSentencePrediction,
                                 BertForSequenceClassification, BertForMultipleChoice,
                                 BertForTokenClassification, BertForQuestionAnswering,
-                                load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+                                load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, Bert2Bert)
     from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel,
                                 OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel,
                                 load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP)
diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index 633c97e263..2a2c3e50ea 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -27,9 +27,9 @@ from .configuration_common_test import ConfigTester
 
 if is_torch_available():
     from transformers import (BertConfig, BertModel, BertForMaskedLM,
-                                        BertForNextSentencePrediction, BertForPreTraining,
-                                        BertForQuestionAnswering, BertForSequenceClassification,
-                                        BertForTokenClassification, BertForMultipleChoice)
+                              BertForNextSentencePrediction, BertForPreTraining,
+                              BertForQuestionAnswering, BertForSequenceClassification,
+                              BertForTokenClassification, BertForMultipleChoice, Bert2Bert)
     from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -38,8 +38,8 @@ else:
 class BertModelTest(CommonTestCases.CommonModelTester):
 
     all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction,
-            BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification,
-            BertForTokenClassification) if is_torch_available() else ()
+                         BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification,
+                         BertForTokenClassification) if is_torch_available() else ()
 
     class BertModelTester(object):
 
@@ -66,7 +66,7 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                      num_labels=3,
                      num_choices=4,
                      scope=None,
-                    ):
+                     ):
             self.parent = parent
             self.batch_size = batch_size
             self.seq_length = seq_length
@@ -145,7 +145,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.hidden_size])
             self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size])
 
-
         def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = BertForMaskedLM(config=config)
             model.eval()
@@ -172,7 +171,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, 2])
             self.check_loss_output(result)
 
-
         def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = BertForPreTraining(config=config)
             model.eval()
@@ -191,7 +189,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, 2])
             self.check_loss_output(result)
 
-
         def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = BertForQuestionAnswering(config=config)
             model.eval()
@@ -210,7 +207,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length])
             self.check_loss_output(result)
 
-
         def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             config.num_labels = self.num_labels
             model = BertForSequenceClassification(config)
@@ -225,7 +221,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
-
         def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             config.num_labels = self.num_labels
             model = BertForTokenClassification(config=config)
@@ -240,7 +235,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.num_labels])
             self.check_loss_output(result)
 
-
         def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             config.num_choices = self.num_choices
             model = BertForMultipleChoice(config=config)
@@ -261,6 +255,16 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.num_choices])
             self.check_loss_output(result)
 
+        def create_and_check_bert2bert(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_choices = self.num_choices
+            model = Bert2Bert(config=config)
+            model.eval()
+            bert2bert_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+            bert2bert_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+            bert2bert_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+            _ = model(bert2bert_inputs_ids,
+                      attention_mask=bert2bert_input_mask,
+                      token_type_ids=bert2bert_token_type_ids)
 
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
@@ -316,5 +320,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
             shutil.rmtree(cache_dir)
             self.assertIsNotNone(model)
 
+
 if __name__ == "__main__":
     unittest.main()

From 8abfee9ec327aea0005a7ad367639217ca7dd215 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 16:07:25 +0200
Subject: [PATCH 013/269] rename Bert2Bert -> Bert2Rnd

---
 transformers/__init__.py                 | 2 +-
 transformers/modeling_bert.py            | 7 ++++---
 transformers/tests/modeling_bert_test.py | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index bf302992b2..006ba9ed16 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -64,7 +64,7 @@ if is_torch_available():
                                 BertForMaskedLM, BertForNextSentencePrediction,
                                 BertForSequenceClassification, BertForMultipleChoice,
                                 BertForTokenClassification, BertForQuestionAnswering,
-                                load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, Bert2Bert)
+                                load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, Bert2Rnd)
     from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel,
                                 OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel,
                                 load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP)
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 9ce32d808e..258e4c3430 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -1419,7 +1419,7 @@ class BertForQuestionAnswering(BertPreTrainedModel):
 @add_start_docstrings("Bert encoder-decoder model for sequence generation.",
                       BERT_START_DOCSTRING,
                       BERT_INPUTS_DOCSTRING)
-class Bert2Bert(BertPreTrainedModel):
+class Bert2Rnd(BertPreTrainedModel):
     r"""
 
     Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
@@ -1434,7 +1434,8 @@ class Bert2Bert(BertPreTrainedModel):
     Examples::
 
         tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-        model = Bert2Bert.from_pretrained('bert-base-uncased')
+        model = Bert2Rnd.from_pretrained('bert-base-uncased')
+        # fine-tuning magic happens here
         input = tokenizer.encode("Hello, how are you?")
         outputs = model(input)
         output_text = tokenize.decode(outputs[0])
@@ -1468,4 +1469,4 @@ class Bert2Bert(BertPreTrainedModel):
                                        token_type_ids=token_type_ids,
                                        position_ids=position_ids,
                                        head_mask=head_mask)
-        return decoder_outputs
+        return decoder_outputs[0]
diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index 2a2c3e50ea..24acf565e3 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -29,7 +29,7 @@ if is_torch_available():
     from transformers import (BertConfig, BertModel, BertForMaskedLM,
                               BertForNextSentencePrediction, BertForPreTraining,
                               BertForQuestionAnswering, BertForSequenceClassification,
-                              BertForTokenClassification, BertForMultipleChoice, Bert2Bert)
+                              BertForTokenClassification, BertForMultipleChoice, Bert2Rnd)
     from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -257,7 +257,7 @@ class BertModelTest(CommonTestCases.CommonModelTester):
 
         def create_and_check_bert2bert(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             config.num_choices = self.num_choices
-            model = Bert2Bert(config=config)
+            model = Bert2Rnd(config=config)
             model.eval()
             bert2bert_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
             bert2bert_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()

From 61ed8890052eb628fe969ed440a38ff82577595c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi@hugginface.co>
Date: Tue, 8 Oct 2019 16:30:07 +0200
Subject: [PATCH 014/269] remove old seq2seq file

---
 transformers/modeling_seq2seq.py            | 39 ---------------------
 transformers/tests/modeling_bert_test.py    |  2 +-
 transformers/tests/modeling_seq2seq_test.py | 23 ------------
 3 files changed, 1 insertion(+), 63 deletions(-)
 delete mode 100644 transformers/modeling_seq2seq.py
 delete mode 100644 transformers/tests/modeling_seq2seq_test.py

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
deleted file mode 100644
index b14622e50f..0000000000
--- a/transformers/modeling_seq2seq.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Conditional generation class. """
-
-
-class Seq2SeqModel(object):
-    def __init__(self):
-        raise EnvironmentError(
-            """Seq2Seq is designed to be instantiated using the
-        `Seq2Seq.from_pretrained(encoder_name_or_path, decoder_name_or_path)` method."""
-        )
-
-    @classmethod
-    def from_pretrained(cls, encoder_name, decoder_name):
-        # Here we should call AutoModel to initialize the models depending
-        # on the pretrained models taken as an input.
-        # For a first iteration we only work with Bert.
-        raise NotImplementedError
-
-    def __call__(self):
-        # allows to call an instance of the class
-        # model = Seq2Seq(encode='bert', decoder='bert')
-        raise NotImplementedError
-
-    def process(self):
-        # alternative API to __call__ it is more explicit.
-        raise NotImplementedError
diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index 24acf565e3..fe9e039983 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -255,7 +255,7 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.num_choices])
             self.check_loss_output(result)
 
-        def create_and_check_bert2bert(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+        def create_and_check_bert2rnd(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             config.num_choices = self.num_choices
             model = Bert2Rnd(config=config)
             model.eval()
diff --git a/transformers/tests/modeling_seq2seq_test.py b/transformers/tests/modeling_seq2seq_test.py
deleted file mode 100644
index 1866dc10af..0000000000
--- a/transformers/tests/modeling_seq2seq_test.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import unittest
-
-
-class Seq2SeqTest(unittest.TestCase):
-    raise NotImplementedError
-
-
-def __main__():
-    unittest.main()

From 770b15b58ceb66a5da72d8030e9aff05fd50848a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Tue, 8 Oct 2019 17:32:28 +0200
Subject: [PATCH 015/269] rename class in __init__

---
 transformers/modeling_bert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 258e4c3430..fc698c772e 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -1449,7 +1449,7 @@ class Bert2Rnd(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(Bert2Bert, self).__init__(config)
+        super(Bert2Rnd, self).__init__(config)
         self.encoder = BertModel(config)
         self.decoder = BertDecoderModel(config)
 

From 851ef592c57bfb0af3807548e798570242c45510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 10:02:03 +0200
Subject: [PATCH 016/269] add comment on recursive weights loading

---
 transformers/modeling_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index 84b64e3ca4..ea114a76fd 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -383,6 +383,8 @@ class PreTrainedModel(nn.Module):
             if metadata is not None:
                 state_dict._metadata = metadata
 
+            # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants
+            # so we need to apply the function recursively.
             def load(module, prefix=''):
                 local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
                 module._load_from_state_dict(

From 877ef2c6cae3059ff9307387baaed886139c5eff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 10:02:18 +0200
Subject: [PATCH 017/269] override `from_pretrained` in Bert2Rnd

In the seq2seq model we need to both load pretrained weights in the
encoder and initialize the decoder randomly. Because the
`from_pretrained` method defined in the base class relies on module
names to assign weights, it would also initialize the decoder with
pretrained weights. To avoid this we override the method to only
initialize the encoder with pretrained weights.
---
 transformers/modeling_bert.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index fc698c772e..db8847f39e 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -1455,6 +1455,37 @@ class Bert2Rnd(BertPreTrainedModel):
 
         self.init_weights()
 
+    @classmethod
+    def from_pretrained(cls, pretrained_model_or_path, *model_args, **model_kwargs):
+        """ Load the pretrained weights in the encoder.
+
+        Since the decoder needs to be initialized with random weights, and the encoder with
+        pretrained weights we need to override the `from_pretrained` method of the base `PreTrainedModel`
+        class.
+        """
+        pretrained_encoder = BertModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
+
+        config = cls._load_config(pretrained_model_or_path, *model_args, **model_kwargs)
+        model = cls(config)
+        model.encoder = pretrained_encoder
+
+        return model
+
+    def _load_config(self, pretrained_model_name_or_path, *args, **kwargs):
+        config = kwargs.pop('config', None)
+        if config is None:
+            cache_dir = kwargs.pop('cache_dir', None)
+            force_download = kwargs.pop('force_download', False)
+            config, _ = self.config_class.from_pretrained(
+                pretrained_model_name_or_path,
+                *args,
+                cache_dir=cache_dir,
+                return_unused_kwargs=True,
+                force_download=force_download,
+                **kwargs
+            )
+        return config
+
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         encoder_outputs = self.encoder(input_ids,
                                        attention_mask=attention_mask,

From 09cfd122353347da7a62eb4f5af75d83b955684f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 10:15:27 +0200
Subject: [PATCH 018/269] remove  and do the branching in

---
 transformers/modeling_bert.py | 68 +++--------------------------------
 1 file changed, 5 insertions(+), 63 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index db8847f39e..94791571cd 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -174,67 +174,6 @@ class BertEmbeddings(nn.Module):
         return embeddings
 
 
-class BertGeneralAttention(nn.Module):
-    def __init__(self, config):
-        super(BertGeneralAttention, self).__init__()
-        if config.hidden_size % config.num_attention_heads != 0:
-            raise ValueError(
-                "The hidden size (%d) is not a multiple of the number of attention "
-                "heads (%d)" % (config.hidden_size, config.num_attention_heads))
-        self.output_attentions = config.output_attentions
-
-        self.num_attention_heads = config.num_attention_heads
-        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
-        self.all_head_size = self.num_attention_heads * self.attention_head_size
-
-        self.query = nn.Linear(config.hidden_size, self.all_head_size)
-        self.key = nn.Linear(config.hidden_size, self.all_head_size)
-        self.value = nn.Linear(config.hidden_size, self.all_head_size)
-
-        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
-
-    def transpose_for_scores(self, x):
-        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
-        x = x.view(*new_x_shape)
-        return x.permute(0, 2, 1, 3)
-
-    def forward(self, query, key, value, attention_mask=None, head_mask=None):
-        mixed_query_layer = self.query(query)
-        mixed_key_layer = self.key(key)
-        mixed_value_layer = self.value(value)
-
-        query_layer = self.transpose_for_scores(mixed_query_layer)
-        key_layer = self.transpose_for_scores(mixed_key_layer)
-        value_layer = self.transpose_for_scores(mixed_value_layer)
-
-        # Take the dot product between "query" and "key" to get the raw attention scores.
-        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
-        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
-        if attention_mask is not None:
-            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
-            attention_scores = attention_scores + attention_mask
-
-        # Normalize the attention scores to probabilities.
-        attention_probs = nn.Softmax(dim=-1)(attention_scores)
-
-        # This is actually dropping out entire tokens to attend to, which might
-        # seem a bit unusual, but is taken from the original Transformer paper.
-        attention_probs = self.dropout(attention_probs)
-
-        # Mask heads if we want to
-        if head_mask is not None:
-            attention_probs = attention_probs * head_mask
-
-        context_layer = torch.matmul(attention_probs, value_layer)
-
-        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
-        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
-        context_layer = context_layer.view(*new_context_layer_shape)
-
-        outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,)
-        return outputs
-
-
 class BertSelfAttention(nn.Module):
     def __init__(self, config):
         super(BertSelfAttention, self).__init__()
@@ -259,10 +198,13 @@ class BertSelfAttention(nn.Module):
         x = x.view(*new_x_shape)
         return x.permute(0, 2, 1, 3)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        mixed_query_layer = self.query(hidden_states)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
         mixed_key_layer = self.key(hidden_states)
         mixed_value_layer = self.value(hidden_states)
+        if encoder_hidden_states:  # if encoder-decoder attention
+            mixed_query_layer = self.query(encoder_hidden_states)
+        else:
+            mixed_query_layer = self.query(hidden_states)
 
         query_layer = self.transpose_for_scores(mixed_query_layer)
         key_layer = self.transpose_for_scores(mixed_key_layer)

From edfc8f822557f3df7d9057a6457a933cddf15299 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 10:17:27 +0200
Subject: [PATCH 019/269] Remove  and do the branching in

---
 transformers/modeling_bert.py | 44 ++---------------------------------
 1 file changed, 2 insertions(+), 42 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 94791571cd..89407ff8ab 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -282,53 +282,13 @@ class BertAttention(nn.Module):
         self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        self_outputs = self.self(hidden_states, attention_mask, head_mask)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
+        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_states)
         attention_output = self.output(self_outputs[0], hidden_states)
         outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
         return outputs
 
 
-class BertDecoderAttention(nn.Module):
-    def __init__(self, config):
-        super(BertAttention, self).__init__()
-        self.self = BertGeneralAttention(config)
-        self.output = BertSelfOutput(config)
-        self.pruned_heads = set()
-
-    def prune_heads(self, heads):
-        if len(heads) == 0:
-            return
-        mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size)
-        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
-        for head in heads:
-            # Compute how many pruned heads are before the head and move the index accordingly
-            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
-            mask[head] = 0
-        mask = mask.view(-1).contiguous().eq(1)
-        index = torch.arange(len(mask))[mask].long()
-
-        # Prune linear layers
-        self.self.query = prune_linear_layer(self.self.query, index)
-        self.self.key = prune_linear_layer(self.self.key, index)
-        self.self.value = prune_linear_layer(self.self.value, index)
-        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
-
-        # Update hyper params and store pruned heads
-        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
-        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
-        self.pruned_heads = self.pruned_heads.union(heads)
-
-    def forward(self, query, key, value, attention_mask=None, head_mask=None):
-        self_outputs = self.self(query, key, value, attention_mask, head_mask)
-        # in encoder-decoder attention we use the output of the previous decoder stage as the query
-        # in the Multi-Head Attention. We thus pass query_tensor as the residual in BertOutput.
-        # This shows the limits of the current code architecture, which may benefit from some refactoring.
-        attention_output = self.output(self_outputs[0], query)
-        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
-        return outputs
-
-
 class BertIntermediate(nn.Module):
     def __init__(self, config):
         super(BertIntermediate, self).__init__()

From 9ca788b2e8f02ea08796e66628b1fd176245f896 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 11:33:28 +0200
Subject: [PATCH 020/269] merge the two Bert layers classes

---
 transformers/modeling_bert.py | 54 +++++++++++++++--------------------
 1 file changed, 23 insertions(+), 31 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 89407ff8ab..f982364f5e 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -318,15 +318,26 @@ class BertOutput(nn.Module):
         return hidden_states
 
 
-class BertEncoderLayer(nn.Module):
+class BertLayer(nn.Module):
     def __init__(self, config):
-        super(BertEncoderLayer, self).__init__()
-        self.attention = BertAttention(config)
+        super(BertLayer, self).__init__()
+        self.self_attention = BertAttention(config)
+        if config.get('is_decoder', False):
+            self.attention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None):
+        self_attention_outputs = self.self_attention(hidden_states, attention_mask, head_mask)
+        self_attention_output = self_attention_outputs[0]
+
+        attention_outputs = self_attention_outputs
+        if encoder_hidden_state:
+            try:
+                attention_outputs = self.attention(self_attention_output, attention_mask, head_mask, encoder_hidden_state)
+            except AttributeError as ae:
+                raise ae("you need to set `is_encoder` to True in the configuration to instantiate an encoder layer")
+
         attention_output = attention_outputs[0]
         intermediate_output = self.intermediate(attention_output)
         layer_output = self.output(intermediate_output, attention_output)
@@ -334,35 +345,12 @@ class BertEncoderLayer(nn.Module):
         return outputs
 
 
-class BertDecoderLayer(nn.Module):
-    def __init__(self, config):
-        super(BertDecoderLayer, self).__init__()
-        self.self_attention = BertAttention(config)
-        self.attention = BertDecoderAttention(config)
-        self.intermediate = BertIntermediate(config)
-        self.output = BertOutput(config)
-
-    def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
-        self_attention_outputs = self.self_attention(hidden_states, attention_mask, head_mask)
-        self_attention_output = self_attention_outputs[0]
-        attention_outputs = self.attention(query=self_attention_output,
-                                           key=encoder_outputs,
-                                           value=encoder_outputs,
-                                           attention_mask=attention_mask,
-                                           head_mask=head_mask)
-        attention_output = attention_outputs[0]
-        intermediate_output = self.intermediate(attention_output)
-        layer_output = self.output(intermediate_output, attention_output)
-        outputs = (layer_output,) + attention_outputs[1:]
-        return outputs
-
-
 class BertEncoder(nn.Module):
     def __init__(self, config):
         super(BertEncoder, self).__init__()
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
-        self.layer = nn.ModuleList([BertEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
         all_hidden_states = ()
@@ -392,9 +380,10 @@ class BertEncoder(nn.Module):
 class BertDecoder(nn.Module):
     def __init__(self, config):
         super(BertDecoder, self).__init__()
+        config["is_decoder"] = True
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
-        self.layers = nn.ModuleList([BertEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.layers = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
 
     def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
         all_hidden_states = ()
@@ -403,7 +392,10 @@ class BertDecoder(nn.Module):
             if self.output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
-            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i])
+            layer_outputs = layer_module(hidden_states,
+                                         attention_mask=attention_mask,
+                                         head_mask=head_mask[i],
+                                         encoder_hidden_state=encoder_outputs)
             if self.output_attentions:
                 all_attentions = all_attentions + (layer_outputs[1],)
 

From df85a0ff0b2847295fde06ab5fd6d2bcb217d59e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 11:38:26 +0200
Subject: [PATCH 021/269] replace double quotes with simple quotes

---
 transformers/modeling_bert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index f982364f5e..a5b21510aa 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -322,7 +322,7 @@ class BertLayer(nn.Module):
     def __init__(self, config):
         super(BertLayer, self).__init__()
         self.self_attention = BertAttention(config)
-        if config.get('is_decoder', False):
+        if config.get("is_decoder", False):
             self.attention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)

From 17177e73796f516e3f49d311eab77b02ab679871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 12:03:58 +0200
Subject: [PATCH 022/269] add is_decoder as an attribute to Config class

---
 transformers/modeling_bert.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index a5b21510aa..9e03c2f8d4 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -322,7 +322,7 @@ class BertLayer(nn.Module):
     def __init__(self, config):
         super(BertLayer, self).__init__()
         self.self_attention = BertAttention(config)
-        if config.get("is_decoder", False):
+        if getattr(config, "is_decoder", False):
             self.attention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
@@ -380,7 +380,7 @@ class BertEncoder(nn.Module):
 class BertDecoder(nn.Module):
     def __init__(self, config):
         super(BertDecoder, self).__init__()
-        config["is_decoder"] = True
+        config.is_decoder = True
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.layers = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])

From 51261167b4a1de53cd38cc2b1553e5d71ba360ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 12:17:22 +0200
Subject: [PATCH 023/269] prune both attention and self-attention heads

---
 transformers/modeling_bert.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 9e03c2f8d4..fddf5d52a2 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -633,7 +633,7 @@ class BertModel(BertPreTrainedModel):
             See base class PreTrainedModel
         """
         for layer, heads in heads_to_prune.items():
-            self.encoder.layer[layer].attention.prune_heads(heads)
+            self.encoder.layer[layer].self_attention.prune_heads(heads)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
@@ -736,7 +736,8 @@ class BertDecoderModel(BertPreTrainedModel):
             See base class PreTrainedModel
         """
         for layer, heads in heads_to_prune.items():
-            self.encoder.layer[layer].attention.prune_heads(heads)
+            self.decoder.layer[layer].attention.prune_heads(heads)
+            self.decoder.layer[layer].self_attention.prune_heads(heads)
 
     def forward(self, input_ids, encoder_outputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:

From d7092d592ca55391b3c07505539b9e4c71bf79de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 12:51:14 +0200
Subject: [PATCH 024/269] rename the attributes in the Bert Layer

Since the preloading of weights relies on the name of the class's
attributes changing the namespace breaks loading pretrained weights on
Bert and all related models. I reverted `self_attention` to `attention`
and us `crossattention` for the decoder instead.
---
 transformers/modeling_bert.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index fddf5d52a2..5fcf41a1e1 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -321,25 +321,24 @@ class BertOutput(nn.Module):
 class BertLayer(nn.Module):
     def __init__(self, config):
         super(BertLayer, self).__init__()
-        self.self_attention = BertAttention(config)
+        self.attention = BertAttention(config)
         if getattr(config, "is_decoder", False):
-            self.attention = BertAttention(config)
+            self.crossattention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None):
-        self_attention_outputs = self.self_attention(hidden_states, attention_mask, head_mask)
-        self_attention_output = self_attention_outputs[0]
+        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
+        attention_output = attention_outputs[0]
 
-        attention_outputs = self_attention_outputs
         if encoder_hidden_state:
             try:
-                attention_outputs = self.attention(self_attention_output, attention_mask, head_mask, encoder_hidden_state)
+                crossattention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
             except AttributeError as ae:
                 raise ae("you need to set `is_encoder` to True in the configuration to instantiate an encoder layer")
 
-        attention_output = attention_outputs[0]
-        intermediate_output = self.intermediate(attention_output)
+        crossattention_output = crossattention_outputs[0]
+        intermediate_output = self.intermediate(crossattention_output)
         layer_output = self.output(intermediate_output, attention_output)
         outputs = (layer_output,) + attention_outputs[1:]  # add attentions if we output them
         return outputs
@@ -633,7 +632,7 @@ class BertModel(BertPreTrainedModel):
             See base class PreTrainedModel
         """
         for layer, heads in heads_to_prune.items():
-            self.encoder.layer[layer].self_attention.prune_heads(heads)
+            self.encoder.layer[layer].attention.prune_heads(heads)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
@@ -737,7 +736,7 @@ class BertDecoderModel(BertPreTrainedModel):
         """
         for layer, heads in heads_to_prune.items():
             self.decoder.layer[layer].attention.prune_heads(heads)
-            self.decoder.layer[layer].self_attention.prune_heads(heads)
+            self.decoder.layer[layer].crossattention.prune_heads(heads)
 
     def forward(self, input_ids, encoder_outputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:

From 81ee29ee8d64c292c3fd5fc7e13b387acd1bfc39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 14:13:37 +0200
Subject: [PATCH 025/269] remove the staticmethod used to load the config

---
 transformers/modeling_bert.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 5fcf41a1e1..6dae6d6ce5 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -715,7 +715,7 @@ class BertDecoderModel(BertPreTrainedModel):
 
     """
     def __init__(self, config):
-        super(BertModel, self).__init__(config)
+        super(BertDecoderModel, self).__init__(config)
 
         self.embeddings = BertEmbeddings(config)
         self.decoder = BertDecoder(config)
@@ -1357,28 +1357,27 @@ class Bert2Rnd(BertPreTrainedModel):
         pretrained weights we need to override the `from_pretrained` method of the base `PreTrainedModel`
         class.
         """
-        pretrained_encoder = BertModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
 
-        config = cls._load_config(pretrained_model_or_path, *model_args, **model_kwargs)
-        model = cls(config)
-        model.encoder = pretrained_encoder
-
-        return model
-
-    def _load_config(self, pretrained_model_name_or_path, *args, **kwargs):
-        config = kwargs.pop('config', None)
+        # Load the configuration
+        config = model_kwargs.pop('config', None)
         if config is None:
-            cache_dir = kwargs.pop('cache_dir', None)
-            force_download = kwargs.pop('force_download', False)
-            config, _ = self.config_class.from_pretrained(
-                pretrained_model_name_or_path,
-                *args,
+            cache_dir = model_kwargs.pop('cache_dir', None)
+            force_download = model_kwargs.pop('force_download', False)
+            config, _ = cls.config_class.from_pretrained(
+                pretrained_model_or_path,
+                *model_args,
                 cache_dir=cache_dir,
                 return_unused_kwargs=True,
                 force_download=force_download,
-                **kwargs
+                **model_kwargs
             )
-        return config
+        model = cls(config)
+
+        # The encoder is loaded with pretrained weights
+        pretrained_encoder = BertModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
+        model.encoder = pretrained_encoder
+
+        return model
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         encoder_outputs = self.encoder(input_ids,

From 3e1cd8241eddc7f3ec036c26f1cbbd3272088653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 14:18:20 +0200
Subject: [PATCH 026/269] fix stupid (re)naming issue

---
 transformers/modeling_bert.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 6dae6d6ce5..5d53b981e5 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -333,12 +333,12 @@ class BertLayer(nn.Module):
 
         if encoder_hidden_state:
             try:
-                crossattention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
+                attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
             except AttributeError as ae:
                 raise ae("you need to set `is_encoder` to True in the configuration to instantiate an encoder layer")
 
-        crossattention_output = crossattention_outputs[0]
-        intermediate_output = self.intermediate(crossattention_output)
+        attention_output = attention_outputs[0]
+        intermediate_output = self.intermediate(attention_output)
         layer_output = self.output(intermediate_output, attention_output)
         outputs = (layer_output,) + attention_outputs[1:]  # add attentions if we output them
         return outputs

From fa218e648abc4f2c2d8a897ed0b4f2f050ecaca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 15:16:07 +0200
Subject: [PATCH 027/269] fix syntax errors

---
 transformers/modeling_bert.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 5d53b981e5..bce7972315 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -201,7 +201,7 @@ class BertSelfAttention(nn.Module):
     def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
         mixed_key_layer = self.key(hidden_states)
         mixed_value_layer = self.value(hidden_states)
-        if encoder_hidden_states:  # if encoder-decoder attention
+        if encoder_hidden_states is not None:  # if encoder-decoder attention
             mixed_query_layer = self.query(encoder_hidden_states)
         else:
             mixed_query_layer = self.query(hidden_states)
@@ -331,11 +331,12 @@ class BertLayer(nn.Module):
         attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
         attention_output = attention_outputs[0]
 
-        if encoder_hidden_state:
+        if encoder_hidden_state is not None:
             try:
                 attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
             except AttributeError as ae:
-                raise ae("you need to set `is_encoder` to True in the configuration to instantiate an encoder layer")
+                print("You need to set `is_encoder` to True in the configuration to instantiate an encoder layer:", ae)
+                raise
 
         attention_output = attention_outputs[0]
         intermediate_output = self.intermediate(attention_output)
@@ -382,7 +383,7 @@ class BertDecoder(nn.Module):
         config.is_decoder = True
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
-        self.layers = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
+        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
 
     def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
         all_hidden_states = ()
@@ -738,7 +739,7 @@ class BertDecoderModel(BertPreTrainedModel):
             self.decoder.layer[layer].attention.prune_heads(heads)
             self.decoder.layer[layer].crossattention.prune_heads(heads)
 
-    def forward(self, input_ids, encoder_outputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+    def forward(self, input_ids, encoder_outputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
         if token_type_ids is None:
@@ -782,7 +783,7 @@ class BertDecoderModel(BertPreTrainedModel):
         sequence_output = decoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+        outputs = (sequence_output, pooled_output,) + decoder_outputs[1:]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
 
 
@@ -1387,8 +1388,7 @@ class Bert2Rnd(BertPreTrainedModel):
                                        head_mask=head_mask)
         encoder_output = encoder_outputs[0]
 
-        decoder_input = torch.empty_like(input_ids).normal_(mean=0.0, std=self.config.initializer_range)
-        decoder_outputs = self.decoder(decoder_input,
+        decoder_outputs = self.decoder(input_ids,
                                        encoder_output,
                                        token_type_ids=token_type_ids,
                                        position_ids=position_ids,

From 1e68c28670cc8d0e8d20ca9fadc697f03908015b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 10 Oct 2019 18:07:11 +0200
Subject: [PATCH 028/269] add test for initialization of Bert2Rnd

---
 examples/run_summarization.py            | 49 ++++++++++++++++++++++++
 transformers/tests/modeling_bert_test.py | 12 +++---
 2 files changed, 55 insertions(+), 6 deletions(-)
 create mode 100644 examples/run_summarization.py

diff --git a/examples/run_summarization.py b/examples/run_summarization.py
new file mode 100644
index 0000000000..0a367551d6
--- /dev/null
+++ b/examples/run_summarization.py
@@ -0,0 +1,49 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning seq2seq models for abstractive summarization.
+
+The finetuning method for abstractive summarization is inspired by [1]. We
+concatenate the document and summary, mask words of the summary at random and
+maximizing the likelihood of masked words.
+
+[1] Dong Li, Nan Yang, Wenhui Wang, Furu Wei, Xiaodong Liu, Yu Wang, Jianfeng
+Gao, Ming Zhou, and Hsiao-Wuen Hon.  “Unified Language Model Pre-Training for
+Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
+"""
+
+import logging
+import random
+
+import numpy as np
+import torch
+
+logger = logging.getLogger(__name__)
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    raise NotImplementedError
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    raise NotImplementedError
diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index fe9e039983..e649cd8ce8 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -259,12 +259,12 @@ class BertModelTest(CommonTestCases.CommonModelTester):
             config.num_choices = self.num_choices
             model = Bert2Rnd(config=config)
             model.eval()
-            bert2bert_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-            bert2bert_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-            bert2bert_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-            _ = model(bert2bert_inputs_ids,
-                      attention_mask=bert2bert_input_mask,
-                      token_type_ids=bert2bert_token_type_ids)
+            bert2rnd_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+            bert2rnd_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+            bert2rnd_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+            _ = model(bert2rnd_inputs_ids,
+                      attention_mask=bert2rnd_input_mask,
+                      token_type_ids=bert2rnd_token_type_ids)
 
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()

From f8e98d67793341f8955634c942af1af579f097dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Fri, 11 Oct 2019 16:48:11 +0200
Subject: [PATCH 029/269] load pretrained embeddings in Bert decoder

In Rothe et al.'s "Leveraging Pre-trained Checkpoints for Sequence
Generation Tasks", Bert2Bert is initialized with pre-trained weights for
the encoder, and only pre-trained embeddings for the decoder. The
current version of the code completely randomizes the weights of the
decoder.

We write a custom function to initiliaze the weights of the decoder; we
first initialize the decoder with the weights and then randomize
everything but the embeddings.
---
 transformers/modeling_bert.py | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index bce7972315..03559ad26c 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -1348,15 +1348,14 @@ class Bert2Rnd(BertPreTrainedModel):
         self.encoder = BertModel(config)
         self.decoder = BertDecoderModel(config)
 
-        self.init_weights()
-
     @classmethod
     def from_pretrained(cls, pretrained_model_or_path, *model_args, **model_kwargs):
         """ Load the pretrained weights in the encoder.
 
-        Since the decoder needs to be initialized with random weights, and the encoder with
-        pretrained weights we need to override the `from_pretrained` method of the base `PreTrainedModel`
-        class.
+        The encoder of `Bert2Rand` is initialized with pretrained weights; the
+        weights of the decoder are initialized at random except the embeddings
+        which are initialized with the pretrained embeddings. We thus need to override
+        the base class' `from_pretrained` method.
         """
 
         # Load the configuration
@@ -1374,10 +1373,26 @@ class Bert2Rnd(BertPreTrainedModel):
             )
         model = cls(config)
 
-        # The encoder is loaded with pretrained weights
+        # We load the encoder with pretrained weights
         pretrained_encoder = BertModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
         model.encoder = pretrained_encoder
 
+        # We load the decoder with pretrained weights and then randomize all weights but embeddings-related one.
+        def randomize_decoder_weights(module):
+            if isinstance(module, nn.Linear):
+                # Slightly different from the TF version which uses truncated_normal for initialization
+                # cf https://github.com/pytorch/pytorch/pull/5617
+                module.weight.data.normal_(mean=0.0, std=config.initializer_range)
+            elif isinstance(module, BertLayerNorm):
+                module.bias.data.zero_()
+                module.weight.data.fill_(1.0)
+            if isinstance(module, nn.Linear) and module.bias is not None:
+                module.bias.data.zero_()
+
+        pretrained_decoder = BertDecoderModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
+        pretrained_decoder.apply(randomize_decoder_weights)
+        model.decoder = pretrained_decoder
+
         return model
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
@@ -1386,11 +1401,9 @@ class Bert2Rnd(BertPreTrainedModel):
                                        token_type_ids=token_type_ids,
                                        position_ids=position_ids,
                                        head_mask=head_mask)
-        encoder_output = encoder_outputs[0]
-
         decoder_outputs = self.decoder(input_ids,
-                                       encoder_output,
+                                       encoder_outputs[0],
                                        token_type_ids=token_type_ids,
                                        position_ids=position_ids,
                                        head_mask=head_mask)
-        return decoder_outputs[0]
+        return decoder_outputs

From d889e0b71beb12511b7fcc346113035e0115ef0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Fri, 11 Oct 2019 17:36:12 +0200
Subject: [PATCH 030/269] add base for seq2seq finetuning

---
 examples/run_seq2seq_finetuning.py | 67 ++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 examples/run_seq2seq_finetuning.py

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
new file mode 100644
index 0000000000..f318bf8036
--- /dev/null
+++ b/examples/run_seq2seq_finetuning.py
@@ -0,0 +1,67 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018 Microsoft and The HuggingFace Inc.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning seq2seq models for sequence generation.
+
+We use the procedure described in [1] to finetune models for sequence
+generation. Let S1 and S2 be the source and target sequence respectively; we
+pack them using the start of sequence [SOS] and end of sequence [EOS] token:
+
+    [SOS] S1 [EOS] S2 [EOS]
+
+We then mask a fixed percentage of token from S2 at random and learn to predict
+the masked words. [EOS] can be masked during finetuning so the model learns to
+terminate the generation process.
+
+[1] Dong Li, Nan Yang, Wenhui Wang, Furu Wei, Xiaodong Liu, Yu Wang, Jianfeng
+Gao, Ming Zhou, and Hsiao-Wuen Hon.  “Unified Language Model Pre-Training for
+Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
+"""
+
+import logging
+import random
+
+import numpy as np
+import torch
+
+logger = logging.getLogger(__name__)
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    """ Fine-tune the pretrained model on the corpus. """
+    # Data sampler
+    # Data loader
+    # Training
+    raise NotImplementedError
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    raise NotImplementedError
+
+
+def main():
+    raise NotImplementedError
+
+
+def __main__():
+    main()

From b3261e7ace153a78c19e35bba367e28e9ccdd2fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Fri, 11 Oct 2019 18:40:38 +0200
Subject: [PATCH 031/269] read parameters from CLI, load model & tokenizer

---
 examples/run_seq2seq_finetuning.py | 60 ++++++++++++++++++++++++------
 examples/run_summarization.py      | 49 ------------------------
 2 files changed, 49 insertions(+), 60 deletions(-)
 delete mode 100644 examples/run_summarization.py

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index f318bf8036..7ad8e4df90 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -30,12 +30,15 @@ Gao, Ming Zhou, and Hsiao-Wuen Hon.  “Unified Language Model Pre-Training for
 Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
 """
 
+import argparse
 import logging
 import random
 
 import numpy as np
 import torch
 
+from transformers import BertConfig, Bert2Rnd, BertTokenizer
+
 logger = logging.getLogger(__name__)
 
 
@@ -43,25 +46,60 @@ def set_seed(args):
     random.seed(args.seed)
     np.random.seed(args.seed)
     torch.manual_seed(args.seed)
-    if args.n_gpu > 0:
-        torch.cuda.manual_seed_all(args.seed)
+
+
+def load_and_cache_examples(args, tokenizer):
+    raise NotImplementedError
 
 
 def train(args, train_dataset, model, tokenizer):
     """ Fine-tune the pretrained model on the corpus. """
-    # Data sampler
-    # Data loader
-    # Training
-    raise NotImplementedError
-
-
-def evaluate(args, model, tokenizer, prefix=""):
     raise NotImplementedError
 
 
 def main():
-    raise NotImplementedError
+    parser = argparse.ArgumentParser()
+
+    # Required parameters
+    parser.add_argument("--train_data_file",
+                        default=None,
+                        type=str,
+                        required=True,
+                        help="The input training data file (a text file).")
+    parser.add_argument("--output_dir",
+                        default=None,
+                        type=str,
+                        required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+
+    # Optional parameters
+    parser.add_argument("--model_name_or_path",
+                        default="bert-base-cased",
+                        type=str,
+                        help="The model checkpoint for weights initialization.")
+    parser.add_argument("--seed", default=42, type=int)
+    args = parser.parse_args()
+
+    # Set up training device
+    device = torch.device("cpu")
+
+    # Set seed
+    set_seed(args)
+
+    # Load pretrained model and tokenizer
+    config_class, model_class, tokenizer_class = BertConfig, Bert2Rnd, BertTokenizer
+    config = config_class.from_pretrained(args.model_name_or_path)
+    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
+    model = model_class.from_pretrained(args.model_name_or_path, config=config)
+    model.to(device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    # Training
+    train_dataset = load_and_cache_examples(args, tokenizer)
+    global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+    logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
 
 
-def __main__():
+if __name__ == "__main__":
     main()
diff --git a/examples/run_summarization.py b/examples/run_summarization.py
deleted file mode 100644
index 0a367551d6..0000000000
--- a/examples/run_summarization.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
-# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Finetuning seq2seq models for abstractive summarization.
-
-The finetuning method for abstractive summarization is inspired by [1]. We
-concatenate the document and summary, mask words of the summary at random and
-maximizing the likelihood of masked words.
-
-[1] Dong Li, Nan Yang, Wenhui Wang, Furu Wei, Xiaodong Liu, Yu Wang, Jianfeng
-Gao, Ming Zhou, and Hsiao-Wuen Hon.  “Unified Language Model Pre-Training for
-Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
-"""
-
-import logging
-import random
-
-import numpy as np
-import torch
-
-logger = logging.getLogger(__name__)
-
-
-def set_seed(args):
-    random.seed(args.seed)
-    np.random.seed(args.seed)
-    torch.manual_seed(args.seed)
-    if args.n_gpu > 0:
-        torch.cuda.manual_seed_all(args.seed)
-
-
-def train(args, train_dataset, model, tokenizer):
-    raise NotImplementedError
-
-
-def evaluate(args, model, tokenizer, prefix=""):
-    raise NotImplementedError

From 0ef9bc923a3fa3f12d39a516aec2069e9ffc4e6e Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 14 Oct 2019 11:58:13 +0200
Subject: [PATCH 032/269] Cleaning up seq2seq [WIP]

---
 transformers/modeling_bert.py    | 284 +++----------------------------
 transformers/modeling_seq2seq.py | 249 +++++++++++++++++++++++++++
 2 files changed, 273 insertions(+), 260 deletions(-)
 create mode 100644 transformers/modeling_seq2seq.py

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 03559ad26c..fbf3c84646 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -199,12 +199,14 @@ class BertSelfAttention(nn.Module):
         return x.permute(0, 2, 1, 3)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
-        mixed_key_layer = self.key(hidden_states)
-        mixed_value_layer = self.value(hidden_states)
-        if encoder_hidden_states is not None:  # if encoder-decoder attention
-            mixed_query_layer = self.query(encoder_hidden_states)
+        mixed_query_layer = self.query(hidden_states)
+        # if the attention Module is a encoder-decoder self attention module
+        if encoder_hidden_states is not None:
+            mixed_key_layer = self.key(encoder_hidden_states)
+            mixed_value_layer = self.value(encoder_hidden_states)
         else:
-            mixed_query_layer = self.query(hidden_states)
+            mixed_key_layer = self.key(hidden_states)
+            mixed_value_layer = self.value(hidden_states)
 
         query_layer = self.transpose_for_scores(mixed_query_layer)
         key_layer = self.transpose_for_scores(mixed_key_layer)
@@ -322,26 +324,25 @@ class BertLayer(nn.Module):
     def __init__(self, config):
         super(BertLayer, self).__init__()
         self.attention = BertAttention(config)
-        if getattr(config, "is_decoder", False):
+        self.is_decoder = config.is_decoder
+        if self.is_decoder:
             self.crossattention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None):
-        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
-        attention_output = attention_outputs[0]
+        self_attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
+        attention_output = self_attention_outputs[0]
+        outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
 
-        if encoder_hidden_state is not None:
-            try:
-                attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
-            except AttributeError as ae:
-                print("You need to set `is_encoder` to True in the configuration to instantiate an encoder layer:", ae)
-                raise
+        if self.is_decoder and encoder_hidden_state is not None:
+            cross_attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
+            attention_output = cross_attention_outputs[0]
+            outputs = outputs + cross_attention_outputs[1:]  # add cross attentions if we output attention weights
 
-        attention_output = attention_outputs[0]
         intermediate_output = self.intermediate(attention_output)
         layer_output = self.output(intermediate_output, attention_output)
-        outputs = (layer_output,) + attention_outputs[1:]  # add attentions if we output them
+        outputs = (layer_output,) + outputs
         return outputs
 
 
@@ -352,14 +353,14 @@ class BertEncoder(nn.Module):
         self.output_hidden_states = config.output_hidden_states
         self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
         all_hidden_states = ()
         all_attentions = ()
         for i, layer_module in enumerate(self.layer):
             if self.output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
-            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i])
+            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_states)
             hidden_states = layer_outputs[0]
 
             if self.output_attentions:
@@ -377,42 +378,6 @@ class BertEncoder(nn.Module):
         return outputs  # last-layer hidden state, (all hidden states), (all attentions)
 
 
-class BertDecoder(nn.Module):
-    def __init__(self, config):
-        super(BertDecoder, self).__init__()
-        config.is_decoder = True
-        self.output_attentions = config.output_attentions
-        self.output_hidden_states = config.output_hidden_states
-        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
-
-    def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
-        all_hidden_states = ()
-        all_attentions = ()
-        for i, layer_module in enumerate(self.layer):
-            if self.output_hidden_states:
-                all_hidden_states = all_hidden_states + (hidden_states,)
-
-            layer_outputs = layer_module(hidden_states,
-                                         attention_mask=attention_mask,
-                                         head_mask=head_mask[i],
-                                         encoder_hidden_state=encoder_outputs)
-            if self.output_attentions:
-                all_attentions = all_attentions + (layer_outputs[1],)
-
-            hidden_states = layer_outputs[0]
-
-        # Add last layer
-        if self.output_hidden_states:
-            all_hidden_states = all_hidden_states + (hidden_states,)
-
-        outputs = (hidden_states,)
-        if self.output_hidden_states:
-            outputs = outputs + (all_hidden_states,)
-        if self.output_attentions:
-            outputs = outputs + (all_attentions,)
-        return outputs  # last-layer hidden state, (all hidden states), (all attentions)
-
-
 class BertPooler(nn.Module):
     def __init__(self, config):
         super(BertPooler, self).__init__()
@@ -635,7 +600,8 @@ class BertModel(BertPreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.encoder.layer[layer].attention.prune_heads(heads)
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None,
+                head_mask=None, encoder_hidden_state=None):
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
         if token_type_ids is None:
@@ -673,8 +639,9 @@ class BertModel(BertPreTrainedModel):
 
         embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
         encoder_outputs = self.encoder(embedding_output,
-                                       extended_attention_mask,
-                                       head_mask=head_mask)
+                                       attention_mask=extended_attention_mask,
+                                       head_mask=head_mask,
+                                       encoder_hidden_state=encoder_hidden_state)
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
@@ -682,111 +649,6 @@ class BertModel(BertPreTrainedModel):
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
 
 
-@add_start_docstrings("""A bare Bert decoder Model transformer outputting raw hidden-states without any specific head on top.
-                      The model follows the general transformer decoder architecture.""",
-                      BERT_START_DOCSTRING,
-                      BERT_INPUTS_DOCSTRING)
-class BertDecoderModel(BertPreTrainedModel):
-    r"""
-    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
-        **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
-            Sequence of hidden-states at the output of the last layer of the model.
-        **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)``
-            Last layer hidden-state of the first token of the sequence (classification token)
-            further processed by a Linear layer and a Tanh activation function. The Linear
-            layer weights are trained from the next sentence prediction (classification)
-            objective during Bert pretraining. This output is usually *not* a good summary
-            of the semantic content of the input, you're often better with averaging or pooling
-            the sequence of hidden-states for the whole input sequence.
-        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
-            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
-            of shape ``(batch_size, sequence_length, hidden_size)``:
-            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
-            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
-            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
-
-    Examples::
-
-        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-        model = BertDecoderModel.from_pretrained('bert-base-uncased')
-        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
-        outputs = model(input_ids)
-        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
-
-    """
-    def __init__(self, config):
-        super(BertDecoderModel, self).__init__(config)
-
-        self.embeddings = BertEmbeddings(config)
-        self.decoder = BertDecoder(config)
-        self.pooler = BertPooler(config)
-
-        self.init_weights()
-
-    def _resize_token_embeddings(self, new_num_tokens):
-        old_embeddings = self.embeddings.word_embeddings
-        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
-        self.embeddings.word_embeddings = new_embeddings
-        return self.embeddings.word_embeddings
-
-    def _prune_heads(self, heads_to_prune):
-        """ Prunes heads of the model.
-            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
-            See base class PreTrainedModel
-        """
-        for layer, heads in heads_to_prune.items():
-            self.decoder.layer[layer].attention.prune_heads(heads)
-            self.decoder.layer[layer].crossattention.prune_heads(heads)
-
-    def forward(self, input_ids, encoder_outputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
-        if attention_mask is None:
-            attention_mask = torch.ones_like(input_ids)
-        if token_type_ids is None:
-            token_type_ids = torch.zeros_like(input_ids)
-
-        # We create a 3D attention mask from a 2D tensor mask.
-        # Sizes are [batch_size, 1, 1, to_seq_length]
-        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
-        # this attention mask is more simple than the triangular masking of causal attention
-        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
-        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
-
-        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
-        # masked positions, this operation will create a tensor which is 0.0 for
-        # positions we want to attend and -10000.0 for masked positions.
-        # Since we are adding it to the raw scores before the softmax, this is
-        # effectively the same as removing these entirely.
-        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
-        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
-
-        # Prepare head mask if needed
-        # 1.0 in head_mask indicate we keep the head
-        # attention_probs has shape bsz x n_heads x N x N
-        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
-        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
-        if head_mask is not None:
-            if head_mask.dim() == 1:
-                head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
-                head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1)
-            elif head_mask.dim() == 2:
-                head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
-            head_mask = head_mask.to(dtype=next(self.parameters()).dtype)  # switch to fload if need + fp16 compatibility
-        else:
-            head_mask = [None] * self.config.num_hidden_layers
-
-        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
-        decoder_outputs = self.decoder(embedding_output,
-                                       encoder_outputs,
-                                       extended_attention_mask,
-                                       head_mask=head_mask)
-        sequence_output = decoder_outputs[0]
-        pooled_output = self.pooler(sequence_output)
-
-        outputs = (sequence_output, pooled_output,) + decoder_outputs[1:]  # add hidden_states and attentions if they are here
-        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
-
-
 @add_start_docstrings("""Bert Model with two heads on top as done during the pre-training:
                        a `masked language modeling` head and a `next sentence prediction (classification)` head. """,
                       BERT_START_DOCSTRING,
@@ -1309,101 +1171,3 @@ class BertForQuestionAnswering(BertPreTrainedModel):
             outputs = (total_loss,) + outputs
 
         return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)
-
-
-@add_start_docstrings("Bert encoder-decoder model for sequence generation.",
-                      BERT_START_DOCSTRING,
-                      BERT_INPUTS_DOCSTRING)
-class Bert2Rnd(BertPreTrainedModel):
-    r"""
-
-    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
-        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
-            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
-            of shape ``(batch_size, sequence_length, hidden_size)``:
-            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
-            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
-            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
-
-    Examples::
-
-        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-        model = Bert2Rnd.from_pretrained('bert-base-uncased')
-        # fine-tuning magic happens here
-        input = tokenizer.encode("Hello, how are you?")
-        outputs = model(input)
-        output_text = tokenize.decode(outputs[0])
-        print(output_text)
-
-    References::
-
-    [1] "Leveraging Pre-trained Checkpoints for Sequence Generation Tasks", S.Rothe, S.Narayan & A.Severyn (2019) ArXiV:1907.12461v1
-    [2] Tensor2Tensor library https://github.com/tensorflow/tensor2tensor
-
-    """
-
-    def __init__(self, config):
-        super(Bert2Rnd, self).__init__(config)
-        self.encoder = BertModel(config)
-        self.decoder = BertDecoderModel(config)
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_or_path, *model_args, **model_kwargs):
-        """ Load the pretrained weights in the encoder.
-
-        The encoder of `Bert2Rand` is initialized with pretrained weights; the
-        weights of the decoder are initialized at random except the embeddings
-        which are initialized with the pretrained embeddings. We thus need to override
-        the base class' `from_pretrained` method.
-        """
-
-        # Load the configuration
-        config = model_kwargs.pop('config', None)
-        if config is None:
-            cache_dir = model_kwargs.pop('cache_dir', None)
-            force_download = model_kwargs.pop('force_download', False)
-            config, _ = cls.config_class.from_pretrained(
-                pretrained_model_or_path,
-                *model_args,
-                cache_dir=cache_dir,
-                return_unused_kwargs=True,
-                force_download=force_download,
-                **model_kwargs
-            )
-        model = cls(config)
-
-        # We load the encoder with pretrained weights
-        pretrained_encoder = BertModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
-        model.encoder = pretrained_encoder
-
-        # We load the decoder with pretrained weights and then randomize all weights but embeddings-related one.
-        def randomize_decoder_weights(module):
-            if isinstance(module, nn.Linear):
-                # Slightly different from the TF version which uses truncated_normal for initialization
-                # cf https://github.com/pytorch/pytorch/pull/5617
-                module.weight.data.normal_(mean=0.0, std=config.initializer_range)
-            elif isinstance(module, BertLayerNorm):
-                module.bias.data.zero_()
-                module.weight.data.fill_(1.0)
-            if isinstance(module, nn.Linear) and module.bias is not None:
-                module.bias.data.zero_()
-
-        pretrained_decoder = BertDecoderModel.from_pretrained(pretrained_model_or_path, *model_args, **model_kwargs)
-        pretrained_decoder.apply(randomize_decoder_weights)
-        model.decoder = pretrained_decoder
-
-        return model
-
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
-        encoder_outputs = self.encoder(input_ids,
-                                       attention_mask=attention_mask,
-                                       token_type_ids=token_type_ids,
-                                       position_ids=position_ids,
-                                       head_mask=head_mask)
-        decoder_outputs = self.decoder(input_ids,
-                                       encoder_outputs[0],
-                                       token_type_ids=token_type_ids,
-                                       position_ids=position_ids,
-                                       head_mask=head_mask)
-        return decoder_outputs
diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
new file mode 100644
index 0000000000..50891ddded
--- /dev/null
+++ b/transformers/modeling_seq2seq.py
@@ -0,0 +1,249 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Auto Model class. """
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging
+
+import torch
+from torch import nn
+
+from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering
+from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel
+from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel
+from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel
+from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering
+from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering
+from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification
+from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification
+
+from .modeling_utils import PreTrainedModel, SequenceSummary
+
+from .file_utils import add_start_docstrings
+
+logger = logging.getLogger(__name__)
+
+
+class PreTrainedSeq2seq(nn.Module):
+    r"""
+        :class:`~transformers.Seq2seq` is a generic model class
+        that will be instantiated as a Seq2seq model with one of the base model classes of the library
+        as encoder and (optionally) as decoder when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)`
+        class method.
+
+        The `from_pretrained()` method takes care of returning the correct model class instance
+        using pattern matching on the `pretrained_model_name_or_path` string.
+
+        The base model class to instantiate is selected as the first pattern matching
+        in the `pretrained_model_name_or_path` string (in the following order):
+            - contains `distilbert`: DistilBertModel (DistilBERT model)
+            - contains `roberta`: RobertaModel (RoBERTa model)
+            - contains `bert`: BertModel (Bert model)
+            - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
+            - contains `gpt2`: GPT2Model (OpenAI GPT-2 model)
+            - contains `transfo-xl`: TransfoXLModel (Transformer-XL model)
+            - contains `xlnet`: XLNetModel (XLNet model)
+            - contains `xlm`: XLMModel (XLM model)
+
+        This class cannot be instantiated using `__init__()` (throws an error).
+    """
+    def __init__(self, encoder, decoder):
+        super(PreTrainedSeq2seq, self).__init__()
+        self.encoder = encoder
+        self.decoder = decoder
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        r""" Instantiates one of the base model classes of the library
+        from a pre-trained model configuration.
+
+        The model class to instantiate is selected as the first pattern matching
+        in the `pretrained_model_name_or_path` string (in the following order):
+            - contains `distilbert`: DistilBertModel (DistilBERT model)
+            - contains `roberta`: RobertaModel (RoBERTa model)
+            - contains `bert`: BertModel (Bert model)
+            - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
+            - contains `gpt2`: GPT2Model (OpenAI GPT-2 model)
+            - contains `transfo-xl`: TransfoXLModel (Transformer-XL model)
+            - contains `xlnet`: XLNetModel (XLNet model)
+            - contains `xlm`: XLMModel (XLM model)
+
+            The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
+            To train the model, you should first set it back in training mode with `model.train()`
+
+        Params:
+            pretrained_model_name_or_path: either:
+
+                - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
+                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
+                - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+
+            model_args: (`optional`) Sequence of positional arguments:
+                All remaning positional arguments will be passed to the underlying model's ``__init__`` method
+
+            config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`:
+                Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when:
+
+                - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or
+                - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory.
+                - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory.
+
+            state_dict: (`optional`) dict:
+                an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file.
+                This option can be used if you want to create a model from a pretrained configuration but load your own weights.
+                In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option.
+
+            cache_dir: (`optional`) string:
+                Path to a directory in which a downloaded pre-trained model
+                configuration should be cached if the standard cache should not be used.
+
+            force_download: (`optional`) boolean, default False:
+                Force to (re-)download the model weights and configuration files and override the cached versions if they exists.
+
+            proxies: (`optional`) dict, default None:
+                A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
+                The proxies are used on each request.
+
+            output_loading_info: (`optional`) boolean:
+                Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
+
+            kwargs: (`optional`) Remaining dictionary of keyword arguments:
+                Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
+
+                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
+                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
+
+        Examples::
+
+            model = AutoModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
+            model = AutoModel.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
+            model = AutoModel.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
+            assert model.config.output_attention == True
+            # Loading from a TF checkpoint file instead of a PyTorch model (slower)
+            config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
+            model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
+
+        """
+        # Extract encoder and decoder model if provided
+        encoder_model = kwargs.pop('encoder_model', None)
+        decoder_model = kwargs.pop('decoder_model', None)
+
+        # Extract decoder kwargs so we only have encoder kwargs for now
+        if decoder_model is None:
+            decoder_pretrained_model_name_or_path = kwargs.pop('decoder_pretrained_model_name_or_path', pretrained_model_name_or_path)
+        decoder_kwargs = {}
+        for key in kwargs.keys():
+            if key.startswith('decoder_'):
+                decoder_kwargs[key.replace('decoder_', '')] = kwargs.pop(key)
+
+        # Load and initialize the decoder
+        if encoder_model:
+            encoder = encoder_model
+        else:
+            # Load and initialize the encoder
+            kwargs['is_decoder'] = False  # Make sure the encoder will be an encoder
+            if 'distilbert' in pretrained_model_name_or_path:
+                encoder = DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'roberta' in pretrained_model_name_or_path:
+                encoder = RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'bert' in pretrained_model_name_or_path:
+                encoder = BertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'openai-gpt' in pretrained_model_name_or_path:
+                encoder = OpenAIGPTModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'gpt2' in pretrained_model_name_or_path:
+                encoder = GPT2Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'transfo-xl' in pretrained_model_name_or_path:
+                encoder = TransfoXLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'xlnet' in pretrained_model_name_or_path:
+                encoder = XLNetModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            elif 'xlm' in pretrained_model_name_or_path:
+                encoder = XLMModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+            else:
+                raise ValueError("Unrecognized model identifier in {}. Should contains one of "
+                                "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
+                                "'xlm', 'roberta'".format(pretrained_model_name_or_path))
+
+        # Load and initialize the decoder
+        if decoder_model:
+            decoder = decoder_model
+        else:
+            kwargs.update(decoder_kwargs)  # Replace encoder kwargs with decoder specific kwargs like config, state_dict, etc...
+            kwargs['is_decoder'] = True  # Make sure the decoder will be an decoder
+            if 'distilbert' in decoder_pretrained_model_name_or_path:
+                decoder = DistilBertModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'roberta' in decoder_pretrained_model_name_or_path:
+                decoder = RobertaModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'bert' in decoder_pretrained_model_name_or_path:
+                decoder = BertModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'openai-gpt' in decoder_pretrained_model_name_or_path:
+                decoder = OpenAIGPTModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'gpt2' in decoder_pretrained_model_name_or_path:
+                decoder = GPT2Model.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'transfo-xl' in decoder_pretrained_model_name_or_path:
+                decoder = TransfoXLModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'xlnet' in decoder_pretrained_model_name_or_path:
+                decoder = XLNetModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            elif 'xlm' in decoder_pretrained_model_name_or_path:
+                decoder = XLMModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            else:
+                raise ValueError("Unrecognized model identifier in {}. Should contains one of "
+                                "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
+                                "'xlm', 'roberta'".format(decoder_pretrained_model_name_or_path))
+
+        model = cls(encoder, decoder)
+        return model
+
+    def forward(self, *inputs, *kwargs):
+        # Extract decoder inputs
+        decoder_kwargs = {}
+        for key in kwargs.keys():
+            if key.startswith('decoder_'):
+                decoder_kwargs[key.replace('decoder_', '')] = kwargs.pop(key)
+
+        # Compute encoder hidden states if needed
+        encoder_hidden_states = kwargs.pop('encoder_hidden_states', None)
+        if encoder_hidden_states is None:
+            encoder_outputs = self.encoder(*inputs, *kwargs)
+            encoder_hidden_states = encoder_outputs[0]
+
+        # Decode
+        decoder_kwargs['encoder_hidden_states'] = encoder_hidden_states
+        decoder_outputs = self.decoder(**decoder_kwargs)
+
+        return decoder_outputs
+
+
+class Model2Model(PreTrainedSeq2seq):
+    def tie_weights():
+        # We should tie encoder and decoder embeddings if possible here
+        pass
+
+
+class Model2LSTM(PreTrainedSeq2seq):
+    @classmethod
+    def from_pretrained(cls, *args, **kwargs):
+        if kwargs.get('decoder_model', None) is None:
+            # We will create a randomly initilized LSTM model as decoder
+            if 'decoder_config' not in kwargs:
+                raise ValueError("To load an LSTM in Seq2seq model, please supply either: "
+                                "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or "
+                                "    - a dictionary of configuration parameters that will be used to initialize a
+                                "        torch.nn.LSTM model as `decoder_config` keyword argument. "
+                                "        E.g. `decoder_config=\{'input_size': 768, 'hidden_size': 768, 'num_layers': 2\}`")
+            kwargs['decoder_model'] = torch.nn.LSTM(kwarg.pop('decoder_config'))
+        model = super(Model2LSTM, cls).from_pretrained(*args, **kwargs)
+        return model
+

From bfbe68f0352a85c0dfff49c5fb0e8296f698f46e Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 14 Oct 2019 12:04:23 +0200
Subject: [PATCH 033/269] update forward pass

---
 transformers/modeling_seq2seq.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 50891ddded..e8106f47f5 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -218,12 +218,14 @@ class PreTrainedSeq2seq(nn.Module):
         if encoder_hidden_states is None:
             encoder_outputs = self.encoder(*inputs, *kwargs)
             encoder_hidden_states = encoder_outputs[0]
+        else:
+            encoder_outputs = (,)
 
         # Decode
         decoder_kwargs['encoder_hidden_states'] = encoder_hidden_states
         decoder_outputs = self.decoder(**decoder_kwargs)
 
-        return decoder_outputs
+        return decoder_outputs + encoder_outputs
 
 
 class Model2Model(PreTrainedSeq2seq):

From b7141a1bc604b8f9512f89d8dc3ec9dcc062e895 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 14 Oct 2019 12:14:08 +0200
Subject: [PATCH 034/269] maxi simplication

---
 transformers/modeling_seq2seq.py | 75 ++------------------------------
 1 file changed, 3 insertions(+), 72 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index e8106f47f5..12792c6e7a 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -21,14 +21,7 @@ import logging
 import torch
 from torch import nn
 
-from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering
-from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel
-from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel
-from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel
-from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering
-from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering
-from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification
-from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification
+from .modeling_auto import AutoModel, AutoModelWithLMHead
 
 from .modeling_utils import PreTrainedModel, SequenceSummary
 
@@ -43,22 +36,6 @@ class PreTrainedSeq2seq(nn.Module):
         that will be instantiated as a Seq2seq model with one of the base model classes of the library
         as encoder and (optionally) as decoder when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)`
         class method.
-
-        The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
-
-        The base model class to instantiate is selected as the first pattern matching
-        in the `pretrained_model_name_or_path` string (in the following order):
-            - contains `distilbert`: DistilBertModel (DistilBERT model)
-            - contains `roberta`: RobertaModel (RoBERTa model)
-            - contains `bert`: BertModel (Bert model)
-            - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
-            - contains `gpt2`: GPT2Model (OpenAI GPT-2 model)
-            - contains `transfo-xl`: TransfoXLModel (Transformer-XL model)
-            - contains `xlnet`: XLNetModel (XLNet model)
-            - contains `xlm`: XLMModel (XLM model)
-
-        This class cannot be instantiated using `__init__()` (throws an error).
     """
     def __init__(self, encoder, decoder):
         super(PreTrainedSeq2seq, self).__init__()
@@ -69,18 +46,6 @@ class PreTrainedSeq2seq(nn.Module):
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
         r""" Instantiates one of the base model classes of the library
         from a pre-trained model configuration.
-
-        The model class to instantiate is selected as the first pattern matching
-        in the `pretrained_model_name_or_path` string (in the following order):
-            - contains `distilbert`: DistilBertModel (DistilBERT model)
-            - contains `roberta`: RobertaModel (RoBERTa model)
-            - contains `bert`: BertModel (Bert model)
-            - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
-            - contains `gpt2`: GPT2Model (OpenAI GPT-2 model)
-            - contains `transfo-xl`: TransfoXLModel (Transformer-XL model)
-            - contains `xlnet`: XLNetModel (XLNet model)
-            - contains `xlm`: XLMModel (XLM model)
-
             The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
             To train the model, you should first set it back in training mode with `model.train()`
 
@@ -155,26 +120,7 @@ class PreTrainedSeq2seq(nn.Module):
         else:
             # Load and initialize the encoder
             kwargs['is_decoder'] = False  # Make sure the encoder will be an encoder
-            if 'distilbert' in pretrained_model_name_or_path:
-                encoder = DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'roberta' in pretrained_model_name_or_path:
-                encoder = RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'bert' in pretrained_model_name_or_path:
-                encoder = BertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'openai-gpt' in pretrained_model_name_or_path:
-                encoder = OpenAIGPTModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'gpt2' in pretrained_model_name_or_path:
-                encoder = GPT2Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'transfo-xl' in pretrained_model_name_or_path:
-                encoder = TransfoXLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'xlnet' in pretrained_model_name_or_path:
-                encoder = XLNetModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            elif 'xlm' in pretrained_model_name_or_path:
-                encoder = XLMModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-            else:
-                raise ValueError("Unrecognized model identifier in {}. Should contains one of "
-                                "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-                                "'xlm', 'roberta'".format(pretrained_model_name_or_path))
+            encoder = AutoModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
 
         # Load and initialize the decoder
         if decoder_model:
@@ -182,22 +128,7 @@ class PreTrainedSeq2seq(nn.Module):
         else:
             kwargs.update(decoder_kwargs)  # Replace encoder kwargs with decoder specific kwargs like config, state_dict, etc...
             kwargs['is_decoder'] = True  # Make sure the decoder will be an decoder
-            if 'distilbert' in decoder_pretrained_model_name_or_path:
-                decoder = DistilBertModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'roberta' in decoder_pretrained_model_name_or_path:
-                decoder = RobertaModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'bert' in decoder_pretrained_model_name_or_path:
-                decoder = BertModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'openai-gpt' in decoder_pretrained_model_name_or_path:
-                decoder = OpenAIGPTModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'gpt2' in decoder_pretrained_model_name_or_path:
-                decoder = GPT2Model.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'transfo-xl' in decoder_pretrained_model_name_or_path:
-                decoder = TransfoXLModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'xlnet' in decoder_pretrained_model_name_or_path:
-                decoder = XLNetModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            elif 'xlm' in decoder_pretrained_model_name_or_path:
-                decoder = XLMModel.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            decoder = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
             else:
                 raise ValueError("Unrecognized model identifier in {}. Should contains one of "
                                 "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "

From d9d387afce183364827da297f2160b84ee43d6fd Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 14 Oct 2019 12:14:40 +0200
Subject: [PATCH 035/269] clean up

---
 transformers/modeling_seq2seq.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 12792c6e7a..466a101f47 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -127,12 +127,8 @@ class PreTrainedSeq2seq(nn.Module):
             decoder = decoder_model
         else:
             kwargs.update(decoder_kwargs)  # Replace encoder kwargs with decoder specific kwargs like config, state_dict, etc...
-            kwargs['is_decoder'] = True  # Make sure the decoder will be an decoder
+            kwargs['is_decoder'] = True  # Make sure the decoder will be a decoder
             decoder = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
-            else:
-                raise ValueError("Unrecognized model identifier in {}. Should contains one of "
-                                "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-                                "'xlm', 'roberta'".format(decoder_pretrained_model_name_or_path))
 
         model = cls(encoder, decoder)
         return model

From 67d10960ae0183b9fa375660ba3ffdd2bb7e959c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Mon, 14 Oct 2019 14:09:21 +0200
Subject: [PATCH 036/269] load and prepare CNN/Daily Mail data

We write a function to load an preprocess the CNN/Daily Mail dataset as
provided by Li Dong et al. The issue is that this dataset has already
been tokenized by the authors, so we actually need to find the original,
plain-text dataset if we want to apply it to all models.
---
 examples/run_seq2seq_finetuning.py | 108 ++++++++++++++++++++++++++++-
 1 file changed, 105 insertions(+), 3 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 7ad8e4df90..7941384506 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -1,5 +1,5 @@
 # coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright 2018 The Microsoft Reseach team and The HuggingFace Inc. team.
 # Copyright (c) 2018 Microsoft and The HuggingFace Inc.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,10 +32,13 @@ Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
 
 import argparse
 import logging
+import pickle
 import random
+import os
 
 import numpy as np
 import torch
+from torch.utils.data import Dataset
 
 from transformers import BertConfig, Bert2Rnd, BertTokenizer
 
@@ -48,8 +51,107 @@ def set_seed(args):
     torch.manual_seed(args.seed)
 
 
+class TextDataset(Dataset):
+    """ Abstracts a dataset used to train seq2seq models.
+
+    A seq2seq dataset consists in two files:
+    - The source file that contains the source sequences, one line per sequence;
+    - The target file contains the target sequences, one line per sequence.
+
+    The matching betwen source and target sequences is made on the basis of line numbers.
+
+    CNN/Daily News:
+
+    The CNN/Daily News dataset downloaded from [1] consists of two files that
+    respectively contain the stories and the associated summaries. Each line
+    corresponds to a different story. The files contain WordPiece tokens.
+
+    train.src: the longest story contains 6966 tokens, the shortest 12.
+    Sentences are separated with `[SEP_i]` where i is an int between 0 and 9.
+
+    train.tgt: the longest summary contains 2467 tokens, the shortest 4.
+    Sentences are separated with `[X_SEP]` tokens.
+
+    [1] https://github.com/microsoft/unilm
+    """
+    def __init_(self, tokenizer, src_path='train.src', target_path='target.src' block_size=512):
+        assert os.path.isfile(file_path)
+        directory, filename = os.path.split(file_path)
+
+        cached_features_file = os.path.join(directory, "cached_lm_{}_{}".format(block_size, file_name)
+        if os.path.exists(cached_features_file):
+            logger.info("Loading features from cached file %s", cached_features_file)
+            with open(cached_features_file, "rb") as source:
+                self.examples = pickle.load(source)
+        else:
+            logger.info("Creating features from dataset at %s", directory)
+
+            self.examples = []
+            with open(src_path, encoding="utf-8") as source, open(target_path, encoding="utf-8") as target:
+                for line_src, line_tgt in zip(source, target)
+                    src_sequence = line_src.read()
+                    tgt_sequence = line_tgt.read()
+                    example = _truncate_and_concatenate(src_sequence, tgt_sequence, block_size)
+                    if example is not None:
+                        example = tokenizer.convert_tokens_to_ids(example)
+                        self.examples.append(example)
+
+            logger.info("Saving features into cache file %s", cached_features_file)
+            with open(cached_features_file, "wb") as sink:
+                pickle.dump(self.examples, sink, protocole=pickle.HIGHEST_PROTOCOL)
+
+    def __len__(self):
+        return len(self.examples)
+
+    def __getitem__(self):
+        return torch.tensor(self.examples[items])
+
+
+def _truncate_and_concatenate(src_sequence, tgt_sequence, block_size):
+    """ Concatenate the sequences and adapt their lengths to the block size.
+
+    Following [1] we perform the following transformations:
+    - Add an [CLS] token at the beginning of the source sequence;
+    - Add an [EOS] token at the end of the source and target sequences;
+    - Concatenate the source and target + tokens sequence. If the concatenated sequence is
+      longer than 512 we follow the 75%/25% rule in [1]: limit the source sequence's length to 384
+      and the target sequence's length to 128.
+
+    [1] Dong, Li, et al. "Unified Language Model Pre-training for Natural
+    Language Understanding and Generation." arXiv preprint arXiv:1905.03197 (2019).
+    """
+    SRC_MAX_LENGTH = int(0.75 * block_size) - 2 # CLS and EOS token
+    TGT_MAX_LENGTH = block_size - SRC_MAX_LENGTH - 1 # EOS token
+
+    # the dataset contains special separator tokens that we remove for now.
+    # They are of the form `[SEP_i]` in the source file, and `[X_SEP]` in the
+    # target file.
+    src_tokens = list(filter(lambda t: "[SEP_" in t, src_sequence.split(" ")))
+    tgt_tokens = list(filter(lambda t: "_SEP]" in t, tgt_sequence.split(" ")))
+
+    # we dump the examples that are too small to fit in the block size for the
+    # sake of simplicity. You can modify this by adding model-specific padding.
+    if len(src_tokens) + len(src_tokens) + 3 < block_size:
+        return None
+
+    # the source sequence has `[SEP_i]` special tokens with i \in [0,9]. We keep them for now.
+    if len(src_tokens) > SRC_MAX_LENGTH
+        if len(tgt_tokens) > TGT_MAX_LENGTH:
+            src_tokens = src_tokens[:SRC_MAX_LENGTH]
+            tgt_tokens = tgt_tokens[:TGT_MAX_LENGTH]
+        else:
+            src_tokens = src_tokens[block_size - len(tgt_tokens) - 3]
+    else:
+        if len(tgt_tokens) > TGT_MAX_LENGTH:
+            tgt_tokens = tgt_tokens[block_size - len(src_tokens) - 3]
+
+    return ["[CLS]"] + src_tokens + ["[EOS]"] + tgt_tokens + ["[EOS]"]
+
+
+
 def load_and_cache_examples(args, tokenizer):
-    raise NotImplementedError
+    dataset = TextDataset(tokenizer, file_path=args.train_data_file)
+    return dataset
 
 
 def train(args, train_dataset, model, tokenizer):
@@ -102,4 +204,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
\ No newline at end of file

From 447fffb21ff41d531b714586e6fac9442594eda2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Mon, 14 Oct 2019 18:12:20 +0200
Subject: [PATCH 037/269] process the raw CNN/Daily Mail dataset

the data provided by Li Dong et al. were already tokenized, which means
that they are not compatible with  all the models in the library. We
thus process the raw data directly and tokenize them using the models'
tokenizers.
---
 examples/run_seq2seq_finetuning.py | 120 ++++++++++++++++++++---------
 1 file changed, 83 insertions(+), 37 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 7941384506..4a7042929e 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -17,9 +17,9 @@
 
 We use the procedure described in [1] to finetune models for sequence
 generation. Let S1 and S2 be the source and target sequence respectively; we
-pack them using the start of sequence [SOS] and end of sequence [EOS] token:
+pack them using the start of sequence [EOS] and end of sequence [EOS] token:
 
-    [SOS] S1 [EOS] S2 [EOS]
+    [CLS] S1 [EOS] S2 [EOS]
 
 We then mask a fixed percentage of token from S2 at random and learn to predict
 the masked words. [EOS] can be masked during finetuning so the model learns to
@@ -31,6 +31,7 @@ Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
 """
 
 import argparse
+import dequeue
 import logging
 import pickle
 import random
@@ -54,7 +55,7 @@ def set_seed(args):
 class TextDataset(Dataset):
     """ Abstracts a dataset used to train seq2seq models.
 
-    A seq2seq dataset consists in two files:
+    A seq2seq dataset consists of two files:
     - The source file that contains the source sequences, one line per sequence;
     - The target file contains the target sequences, one line per sequence.
 
@@ -62,43 +63,53 @@ class TextDataset(Dataset):
 
     CNN/Daily News:
 
-    The CNN/Daily News dataset downloaded from [1] consists of two files that
-    respectively contain the stories and the associated summaries. Each line
-    corresponds to a different story. The files contain WordPiece tokens.
+    The CNN/Daily News raw datasets are downloaded from [1]. They consist in stories stored
+    in different files where the summary sentences are indicated by the special `@highlight` token.
+    To process the data, untar both datasets in the same folder, and path the path to this
+    folder as the "train_data_file" argument. The formatting code was inspired by [2].
 
-    train.src: the longest story contains 6966 tokens, the shortest 12.
-    Sentences are separated with `[SEP_i]` where i is an int between 0 and 9.
-
-    train.tgt: the longest summary contains 2467 tokens, the shortest 4.
-    Sentences are separated with `[X_SEP]` tokens.
-
-    [1] https://github.com/microsoft/unilm
+    [1] https://cs.nyu.edu/~kcho/
+    [2] https://github.com/abisee/cnn-dailymail/
     """
-    def __init_(self, tokenizer, src_path='train.src', target_path='target.src' block_size=512):
-        assert os.path.isfile(file_path)
-        directory, filename = os.path.split(file_path)
+    def __init_(self, tokenizer, data_dir='', block_size=512):
+        assert os.path.isdir(data_dir)
 
-        cached_features_file = os.path.join(directory, "cached_lm_{}_{}".format(block_size, file_name)
+        # Load features that have already been computed if present
+        cached_features_file = os.path.join(directory, "cached_lm_{}_{}".format(block_size, data_dir)
         if os.path.exists(cached_features_file):
             logger.info("Loading features from cached file %s", cached_features_file)
             with open(cached_features_file, "rb") as source:
                 self.examples = pickle.load(source)
-        else:
-            logger.info("Creating features from dataset at %s", directory)
+                return
 
-            self.examples = []
-            with open(src_path, encoding="utf-8") as source, open(target_path, encoding="utf-8") as target:
-                for line_src, line_tgt in zip(source, target)
-                    src_sequence = line_src.read()
-                    tgt_sequence = line_tgt.read()
-                    example = _truncate_and_concatenate(src_sequence, tgt_sequence, block_size)
-                    if example is not None:
-                        example = tokenizer.convert_tokens_to_ids(example)
-                        self.examples.append(example)
+        logger.info("Creating features from dataset at %s", directory)
 
-            logger.info("Saving features into cache file %s", cached_features_file)
-            with open(cached_features_file, "wb") as sink:
-                pickle.dump(self.examples, sink, protocole=pickle.HIGHEST_PROTOCOL)
+        # we need to iterate over both the cnn and the dailymail dataset
+        datasets = ['cnn', 'dailymail']
+        for dataset in datasets:
+            path_to_stories = os.path.join(data_dir, dataset, "stories")
+            assert os.path.isdir(path_to_stories)
+
+            stories_files = os.listdir(path_to_stories)
+            for story_file in stories_files:
+                path_to_story = os.path.join(path_to_stories, "story_file")
+                if !os.path.isfile(path_to_story):
+                    continue
+
+                with open(path_to_story, encoding="utf-8") as source:
+                    try:
+                        story, summary = process_story(source)
+                    except IndexError:
+                        continue
+
+                src_sequence = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
+                tgt_sequence = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
+                example = _truncate_and_concatenate(src_sequence, tgt_sequence, blocksize)
+                self.examples.append(example)
+
+        logger.info("Saving features into cache file %s", cached_features_file)
+        with open(cached_features_file, "wb") as sink:
+            pickle.dump(self.examples, sink, protocole=pickle.HIGHEST_PROTOCOL)
 
     def __len__(self):
         return len(self.examples)
@@ -107,6 +118,46 @@ class TextDataset(Dataset):
         return torch.tensor(self.examples[items])
 
 
+def process_story(story_file):
+    """ Process the text contained in a story file.
+    Returns the story and the summary
+    """
+    file_lines = list(filter(lambda x: len(x)!=0, [line.strip() for lines in story_file]))
+
+    # for some unknown reason some lines miss a period, add it
+    file_lines = [_add_missing_period(line) for line in file_lines]
+
+    # gather article lines
+    story_lines = []
+    lines = dequeue(file_lines)
+    while True:
+        try:
+            element = lines.popleft()
+            if element.startswith("@highlight"):
+                break
+            story_lines.append(element)
+        except IndexError as ie:  # if "@highlight" absent from file
+            raise ie
+
+    # gather summary lines
+    highlights_lines = list(filter(lambda t: !t.startswith("@highlight"), lines))
+
+    # join the lines
+    story = " ".join(story_lines)
+    summary = " ".join(highlights_lines)
+
+    return story, summary
+
+
+def _add_missing_period(line):
+    END_TOKENS = ['.', '!', '?', '...', "'", "`", '"', u'\u2019', u'\u2019', ")"]
+    if line == "@highlight":
+        return line
+    if line[-1] in END_TOKENS:
+        return line
+    return line + " ."
+
+
 def _truncate_and_concatenate(src_sequence, tgt_sequence, block_size):
     """ Concatenate the sequences and adapt their lengths to the block size.
 
@@ -123,12 +174,6 @@ def _truncate_and_concatenate(src_sequence, tgt_sequence, block_size):
     SRC_MAX_LENGTH = int(0.75 * block_size) - 2 # CLS and EOS token
     TGT_MAX_LENGTH = block_size - SRC_MAX_LENGTH - 1 # EOS token
 
-    # the dataset contains special separator tokens that we remove for now.
-    # They are of the form `[SEP_i]` in the source file, and `[X_SEP]` in the
-    # target file.
-    src_tokens = list(filter(lambda t: "[SEP_" in t, src_sequence.split(" ")))
-    tgt_tokens = list(filter(lambda t: "_SEP]" in t, tgt_sequence.split(" ")))
-
     # we dump the examples that are too small to fit in the block size for the
     # sake of simplicity. You can modify this by adding model-specific padding.
     if len(src_tokens) + len(src_tokens) + 3 < block_size:
@@ -145,6 +190,7 @@ def _truncate_and_concatenate(src_sequence, tgt_sequence, block_size):
         if len(tgt_tokens) > TGT_MAX_LENGTH:
             tgt_tokens = tgt_tokens[block_size - len(src_tokens) - 3]
 
+    # I add the special tokens manually, but this should be done by the tokenizer. That's the next step.
     return ["[CLS]"] + src_tokens + ["[EOS]"] + tgt_tokens + ["[EOS]"]
 
 

From 412793275d3773ef0aab0e17b76a4010e7082656 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Mon, 14 Oct 2019 20:45:16 +0200
Subject: [PATCH 038/269] delegate the padding with special tokens to the
 tokenizer

---
 examples/run_seq2seq_finetuning.py | 53 +++++++++++++-----------------
 1 file changed, 22 insertions(+), 31 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 4a7042929e..5d7da58a23 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -53,20 +53,14 @@ def set_seed(args):
 
 
 class TextDataset(Dataset):
-    """ Abstracts a dataset used to train seq2seq models.
-
-    A seq2seq dataset consists of two files:
-    - The source file that contains the source sequences, one line per sequence;
-    - The target file contains the target sequences, one line per sequence.
-
-    The matching betwen source and target sequences is made on the basis of line numbers.
+    """ Abstracts the dataset used to train seq2seq models.
 
     CNN/Daily News:
 
     The CNN/Daily News raw datasets are downloaded from [1]. They consist in stories stored
     in different files where the summary sentences are indicated by the special `@highlight` token.
-    To process the data, untar both datasets in the same folder, and path the path to this
-    folder as the "train_data_file" argument. The formatting code was inspired by [2].
+    To process the data, untar both datasets in the same folder, and pass the path to this
+    folder as the "data_dir argument. The formatting code was inspired by [2].
 
     [1] https://cs.nyu.edu/~kcho/
     [2] https://github.com/abisee/cnn-dailymail/
@@ -82,9 +76,8 @@ class TextDataset(Dataset):
                 self.examples = pickle.load(source)
                 return
 
-        logger.info("Creating features from dataset at %s", directory)
+        logger.info("Creating features from dataset at %s", data_dir)
 
-        # we need to iterate over both the cnn and the dailymail dataset
         datasets = ['cnn', 'dailymail']
         for dataset in datasets:
             path_to_stories = os.path.join(data_dir, dataset, "stories")
@@ -102,9 +95,10 @@ class TextDataset(Dataset):
                     except IndexError:
                         continue
 
-                src_sequence = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
-                tgt_sequence = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
-                example = _truncate_and_concatenate(src_sequence, tgt_sequence, blocksize)
+                story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
+                summary = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
+                story_seq, summary_seq = _fit_to_block_size(story, summary, blocksize)
+                example = tokenizer.add_special_token_sequence_pair(story_seq, summary_seq)
                 self.examples.append(example)
 
         logger.info("Saving features into cache file %s", cached_features_file)
@@ -158,15 +152,13 @@ def _add_missing_period(line):
     return line + " ."
 
 
-def _truncate_and_concatenate(src_sequence, tgt_sequence, block_size):
+def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
     """ Concatenate the sequences and adapt their lengths to the block size.
 
-    Following [1] we perform the following transformations:
-    - Add an [CLS] token at the beginning of the source sequence;
-    - Add an [EOS] token at the end of the source and target sequences;
-    - Concatenate the source and target + tokens sequence. If the concatenated sequence is
-      longer than 512 we follow the 75%/25% rule in [1]: limit the source sequence's length to 384
-      and the target sequence's length to 128.
+    Following [1] we truncate the source and target + tokens sequences so they fit
+    in the block size. If the concatenated sequence is longer than 512 we follow
+    the 75%/25% rule in [1]: limit the source sequence's length to 384 and the
+    target sequence's length to 128.
 
     [1] Dong, Li, et al. "Unified Language Model Pre-training for Natural
     Language Understanding and Generation." arXiv preprint arXiv:1905.03197 (2019).
@@ -176,22 +168,21 @@ def _truncate_and_concatenate(src_sequence, tgt_sequence, block_size):
 
     # we dump the examples that are too small to fit in the block size for the
     # sake of simplicity. You can modify this by adding model-specific padding.
-    if len(src_tokens) + len(src_tokens) + 3 < block_size:
+    if len(src_sequence) + len(src_sequence) + 3 < block_size:
         return None
 
     # the source sequence has `[SEP_i]` special tokens with i \in [0,9]. We keep them for now.
-    if len(src_tokens) > SRC_MAX_LENGTH
-        if len(tgt_tokens) > TGT_MAX_LENGTH:
-            src_tokens = src_tokens[:SRC_MAX_LENGTH]
-            tgt_tokens = tgt_tokens[:TGT_MAX_LENGTH]
+    if len(src_sequence) > SRC_MAX_LENGTH
+        if len(tgt_sequence) > TGT_MAX_LENGTH:
+            src_sequence = src_sequence[:SRC_MAX_LENGTH]
+            tgt_sequence = tgt_sequence[:TGT_MAX_LENGTH]
         else:
-            src_tokens = src_tokens[block_size - len(tgt_tokens) - 3]
+            src_sequence = src_sequence[block_size - len(tgt_sequence) - 3]
     else:
         if len(tgt_tokens) > TGT_MAX_LENGTH:
-            tgt_tokens = tgt_tokens[block_size - len(src_tokens) - 3]
+            tgt_sequence = tgt_sequence[block_size - len(src_sequence) - 3]
 
-    # I add the special tokens manually, but this should be done by the tokenizer. That's the next step.
-    return ["[CLS]"] + src_tokens + ["[EOS]"] + tgt_tokens + ["[EOS]"]
+    return src_sequence, tgt_sequence
 
 
 
@@ -250,4 +241,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()

From fe25eefc1589a0362e1b60c30734f88f666aff5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Mon, 14 Oct 2019 20:45:39 +0200
Subject: [PATCH 039/269] add instructions to fetch the dataset

---
 examples/README.md | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/examples/README.md b/examples/README.md
index fb5de20a2a..ba58a61012 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -10,6 +10,7 @@ similar API between the different models.
 | [GLUE](#glue) | Examples running BERT/XLM/XLNet/RoBERTa on the 9 GLUE tasks. Examples feature distributed training as well as half-precision.                              |
 | [SQuAD](#squad) | Using BERT for question answering, examples with distributed training.                                                                                  |
 | [Multiple Choice](#multiple choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. 
+| [Seq2seq Model fine-tuning](#seq2seq-model-fine-tuning) | Fine-tuning the library models for seq2seq tasks on the CNN/Daily Mail dataset. |
 
 ## Language model fine-tuning
 
@@ -387,6 +388,30 @@ f1 = 93.15
 exact_match = 86.91
 ```
 
-This fine-tuneds model is available as a checkpoint under the reference
+This fine-tuned model is available as a checkpoint under the reference
 `bert-large-uncased-whole-word-masking-finetuned-squad`.
 
+## Seq2seq model fine-tuning
+
+Based on the script [`run_seq2seq_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_seq2seq_finetuning.py).
+
+Before running this script you should download **both** CNN and Daily Mail datasets (the links next to "Stories") from [Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/) in the same folder. Then uncompress the archives by running:
+
+```bash
+tar -xvf cnn_stories.tgz && tar -xvf dailymail_stories.tgz
+```
+
+We will refer as `$DATA_PATH` the path to where you uncompressed both archive.
+
+## Bert2Bert and abstractive summarization
+
+```bash
+export DATA_PATH=/path/to/dataset/
+
+python run_seq2seq_finetuning.py \
+    --output_dir=output \
+    --model_type=bert2bert \
+    --model_name_or_path=bert2bert \
+    --do_train \
+    --data_path=$DATA_PATH \
+```
\ No newline at end of file

From 74c5035808de18b016c88b1f864d609bc684b367 Mon Sep 17 00:00:00 2001
From: hlums <honglu@microsoft.com>
Date: Mon, 14 Oct 2019 21:27:11 +0000
Subject: [PATCH 040/269] Fix token order in xlnet preprocessing.

---
 examples/run_squad.py   |  6 +++++-
 examples/utils_squad.py | 41 ++++++++++++++++++++++++++++++-----------
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/examples/run_squad.py b/examples/run_squad.py
index 43b65d2c3c..a746d441df 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -302,7 +302,11 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal
                                                 max_seq_length=args.max_seq_length,
                                                 doc_stride=args.doc_stride,
                                                 max_query_length=args.max_query_length,
-                                                is_training=not evaluate)
+                                                is_training=not evaluate,
+                                                cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0,
+                                                pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0,
+                                                cls_token_at_end=True if args.model_type in ['xlnet'] else False,
+                                                sequence_a_is_doc=True if args.model_type in ['xlnet'] else False)
         if args.local_rank in [-1, 0]:
             logger.info("Saving features into cached file %s", cached_features_file)
             torch.save(features, cached_features_file)
diff --git a/examples/utils_squad.py b/examples/utils_squad.py
index b990ecc842..6d1c86493d 100644
--- a/examples/utils_squad.py
+++ b/examples/utils_squad.py
@@ -192,7 +192,8 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                  cls_token='[CLS]', sep_token='[SEP]', pad_token=0,
                                  sequence_a_segment_id=0, sequence_b_segment_id=1,
                                  cls_token_segment_id=0, pad_token_segment_id=0,
-                                 mask_padding_with_zero=True):
+                                 mask_padding_with_zero=True,
+                                 sequence_a_is_doc=False):
     """Loads a data file into a list of `InputBatch`s."""
 
     unique_id = 1000000000
@@ -272,17 +273,19 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
                 p_mask.append(0)
                 cls_index = 0
 
-            # Query
-            for token in query_tokens:
-                tokens.append(token)
+            # XLNet: P SEP Q SEP CLS
+            # Others: CLS Q SEP P SEP
+            if not sequence_a_is_doc:
+                # Query
+                tokens += query_tokens
+                segment_ids += [sequence_a_segment_id] * len(query_tokens)
+                p_mask += [1] * len(query_tokens)
+
+                # SEP token
+                tokens.append(sep_token)
                 segment_ids.append(sequence_a_segment_id)
                 p_mask.append(1)
 
-            # SEP token
-            tokens.append(sep_token)
-            segment_ids.append(sequence_a_segment_id)
-            p_mask.append(1)
-
             # Paragraph
             for i in range(doc_span.length):
                 split_token_index = doc_span.start + i
@@ -292,10 +295,23 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                                        split_token_index)
                 token_is_max_context[len(tokens)] = is_max_context
                 tokens.append(all_doc_tokens[split_token_index])
-                segment_ids.append(sequence_b_segment_id)
+                if not sequence_a_is_doc:
+                    segment_ids.append(sequence_b_segment_id)
+                else:
+                    segment_ids.append(sequence_a_segment_id)
                 p_mask.append(0)
             paragraph_len = doc_span.length
 
+            if sequence_a_is_doc:
+                # SEP token
+                tokens.append(sep_token)
+                segment_ids.append(sequence_a_segment_id)
+                p_mask.append(1)
+
+                tokens += query_tokens
+                segment_ids += [sequence_b_segment_id] * len(query_tokens)
+                p_mask += [1] * len(query_tokens)
+
             # SEP token
             tokens.append(sep_token)
             segment_ids.append(sequence_b_segment_id)
@@ -342,7 +358,10 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
                     end_position = 0
                     span_is_impossible = True
                 else:
-                    doc_offset = len(query_tokens) + 2
+                    if sequence_a_is_doc:
+                        doc_offset = 0
+                    else:
+                        doc_offset = len(query_tokens) + 2
                     start_position = tok_start_position - doc_start + doc_offset
                     end_position = tok_end_position - doc_start + doc_offset
 

From 260ac7d9a8501f6c631adc355e269e7f3f6274f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Tue, 15 Oct 2019 12:24:35 +0200
Subject: [PATCH 041/269] wip commit, switching computers

---
 examples/run_seq2seq_finetuning.py      | 42 ++++++++--------
 examples/run_seq2seq_finetuning_test.py | 64 +++++++++++++++++++++++++
 2 files changed, 85 insertions(+), 21 deletions(-)
 create mode 100644 examples/run_seq2seq_finetuning_test.py

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 5d7da58a23..1f247ab25b 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -31,7 +31,7 @@ Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
 """
 
 import argparse
-import dequeue
+from collections import deque
 import logging
 import pickle
 import random
@@ -57,9 +57,9 @@ class TextDataset(Dataset):
 
     CNN/Daily News:
 
-    The CNN/Daily News raw datasets are downloaded from [1]. They consist in stories stored
-    in different files where the summary sentences are indicated by the special `@highlight` token.
-    To process the data, untar both datasets in the same folder, and pass the path to this
+    The CNN/Daily News raw datasets are downloaded from [1]. The stories are stored in different files; the summary appears at the end of the story as
+    sentences that are prefixed by the special `@highlight` line. To process the
+    data, untar both datasets in the same folder, and pass the path to this
     folder as the "data_dir argument. The formatting code was inspired by [2].
 
     [1] https://cs.nyu.edu/~kcho/
@@ -69,7 +69,7 @@ class TextDataset(Dataset):
         assert os.path.isdir(data_dir)
 
         # Load features that have already been computed if present
-        cached_features_file = os.path.join(directory, "cached_lm_{}_{}".format(block_size, data_dir)
+        cached_features_file = os.path.join(directory, "cached_lm_{}_{}".format(block_size, data_dir))
         if os.path.exists(cached_features_file):
             logger.info("Loading features from cached file %s", cached_features_file)
             with open(cached_features_file, "rb") as source:
@@ -86,18 +86,19 @@ class TextDataset(Dataset):
             stories_files = os.listdir(path_to_stories)
             for story_file in stories_files:
                 path_to_story = os.path.join(path_to_stories, "story_file")
-                if !os.path.isfile(path_to_story):
+                if not os.path.isfile(path_to_story):
                     continue
 
                 with open(path_to_story, encoding="utf-8") as source:
                     try:
-                        story, summary = process_story(source)
+                        raw_story = source.read()
+                        story, summary = process_story(raw_story)
                     except IndexError:
                         continue
 
                 story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
                 summary = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
-                story_seq, summary_seq = _fit_to_block_size(story, summary, blocksize)
+                story_seq, summary_seq = _fit_to_block_size(story, summary, block_size)
                 example = tokenizer.add_special_token_sequence_pair(story_seq, summary_seq)
                 self.examples.append(example)
 
@@ -108,22 +109,22 @@ class TextDataset(Dataset):
     def __len__(self):
         return len(self.examples)
 
-    def __getitem__(self):
+    def __getitem__(self, items):
         return torch.tensor(self.examples[items])
 
 
-def process_story(story_file):
+def process_story(raw_story):
     """ Process the text contained in a story file.
     Returns the story and the summary
     """
-    file_lines = list(filter(lambda x: len(x)!=0, [line.strip() for lines in story_file]))
+    file_lines = list(filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")]))
 
     # for some unknown reason some lines miss a period, add it
     file_lines = [_add_missing_period(line) for line in file_lines]
 
     # gather article lines
     story_lines = []
-    lines = dequeue(file_lines)
+    lines = deque(file_lines)
     while True:
         try:
             element = lines.popleft()
@@ -134,7 +135,7 @@ def process_story(story_file):
             raise ie
 
     # gather summary lines
-    highlights_lines = list(filter(lambda t: !t.startswith("@highlight"), lines))
+    highlights_lines = list(filter(lambda t: not t.startswith("@highlight"), lines))
 
     # join the lines
     story = " ".join(story_lines)
@@ -145,7 +146,7 @@ def process_story(story_file):
 
 def _add_missing_period(line):
     END_TOKENS = ['.', '!', '?', '...', "'", "`", '"', u'\u2019', u'\u2019', ")"]
-    if line == "@highlight":
+    if line.startswith("@highlight"):
         return line
     if line[-1] in END_TOKENS:
         return line
@@ -163,8 +164,8 @@ def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
     [1] Dong, Li, et al. "Unified Language Model Pre-training for Natural
     Language Understanding and Generation." arXiv preprint arXiv:1905.03197 (2019).
     """
-    SRC_MAX_LENGTH = int(0.75 * block_size) - 2 # CLS and EOS token
-    TGT_MAX_LENGTH = block_size - SRC_MAX_LENGTH - 1 # EOS token
+    SRC_MAX_LENGTH = int(0.75 * block_size) - 2  # CLS and EOS token
+    TGT_MAX_LENGTH = block_size - SRC_MAX_LENGTH - 1  # EOS token
 
     # we dump the examples that are too small to fit in the block size for the
     # sake of simplicity. You can modify this by adding model-specific padding.
@@ -172,22 +173,21 @@ def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
         return None
 
     # the source sequence has `[SEP_i]` special tokens with i \in [0,9]. We keep them for now.
-    if len(src_sequence) > SRC_MAX_LENGTH
+    if len(src_sequence) > SRC_MAX_LENGTH:
         if len(tgt_sequence) > TGT_MAX_LENGTH:
             src_sequence = src_sequence[:SRC_MAX_LENGTH]
             tgt_sequence = tgt_sequence[:TGT_MAX_LENGTH]
         else:
             src_sequence = src_sequence[block_size - len(tgt_sequence) - 3]
     else:
-        if len(tgt_tokens) > TGT_MAX_LENGTH:
+        if len(tgt_sequence) > TGT_MAX_LENGTH:
             tgt_sequence = tgt_sequence[block_size - len(src_sequence) - 3]
 
     return src_sequence, tgt_sequence
 
 
-
 def load_and_cache_examples(args, tokenizer):
-    dataset = TextDataset(tokenizer, file_path=args.train_data_file)
+    dataset = TextDataset(tokenizer, file_path=args.data_dir)
     return dataset
 
 
@@ -200,7 +200,7 @@ def main():
     parser = argparse.ArgumentParser()
 
     # Required parameters
-    parser.add_argument("--train_data_file",
+    parser.add_argument("--data_dir",
                         default=None,
                         type=str,
                         required=True,
diff --git a/examples/run_seq2seq_finetuning_test.py b/examples/run_seq2seq_finetuning_test.py
new file mode 100644
index 0000000000..34d9add10d
--- /dev/null
+++ b/examples/run_seq2seq_finetuning_test.py
@@ -0,0 +1,64 @@
+# coding=utf-8
+# Copyright 2019 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+from .run_seq2seq_finetuning import process_story, _fit_to_block_size
+
+
+class DataLoaderTest(unittest.TestCase):
+    def __init__(self, block_size=10):
+        self.block_size = block_size
+
+    def source_and_target_too_small(self):
+        """ When the sum of the lengths of the source and target sequences is
+        smaller than the block size (minus the number of special tokens), skip the example. """
+        src_seq = [1, 2, 3, 4]
+        tgt_seq = [5, 6]
+        self.assertEqual(_fit_to_block_size(src_seq, tgt_seq, self.block_size), None)
+
+    def source_and_target_fit_exactly(self):
+        """ When the sum of the lengths of the source and target sequences is
+        equal to the block size (minus the number of special tokens), return the
+        sequences unchanged. """
+        src_seq = [1, 2, 3, 4]
+        tgt_seq = [5, 6, 7]
+        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
+        self.assertListEqual(src_seq == fitted_src)
+        self.assertListEqual(tgt_seq == fitted_tgt)
+
+    def source_too_big_target_ok(self):
+        src_seq = [1, 2, 3, 4, 5, 6]
+        tgt_seq = [1, 2]
+        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
+        self.assertListEqual(src_seq == [1, 2, 3, 4, 5])
+        self.assertListEqual(tgt_seq == fitted_tgt)
+
+    def target_too_big_source_ok(self):
+        src_seq = [1, 2, 3, 4]
+        tgt_seq = [1, 2, 3, 4]
+        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
+        self.assertListEqual(src_seq == src_seq)
+        self.assertListEqual(tgt_seq == [1, 2, 3])
+
+    def source_and_target_too_big(self):
+        src_seq = [1, 2, 3, 4, 5, 6, 7]
+        tgt_seq = [1, 2, 3, 4, 5, 6, 7]
+        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
+        self.assertListEqual(src_seq == [1, 2, 3, 4, 5])
+        self.assertListEqual(tgt_seq == [1, 2])
+
+
+if __name__ == "__main__":
+    unittest.main()

From 22e1af68596690558cd8df45b6bc75e665cc1c1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 14:39:56 +0200
Subject: [PATCH 042/269] truncation function is fully tested

---
 examples/run_seq2seq_finetuning.py      | 101 ++++++++++++++----------
 examples/run_seq2seq_finetuning_test.py |  32 ++++----
 2 files changed, 74 insertions(+), 59 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 1f247ab25b..e926523a17 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -41,7 +41,7 @@ import numpy as np
 import torch
 from torch.utils.data import Dataset
 
-from transformers import BertConfig, Bert2Rnd, BertTokenizer
+from transformers import BertTokenizer
 
 logger = logging.getLogger(__name__)
 
@@ -57,19 +57,23 @@ class TextDataset(Dataset):
 
     CNN/Daily News:
 
-    The CNN/Daily News raw datasets are downloaded from [1]. The stories are stored in different files; the summary appears at the end of the story as
-    sentences that are prefixed by the special `@highlight` line. To process the
-    data, untar both datasets in the same folder, and pass the path to this
+    The CNN/Daily News raw datasets are downloaded from [1]. The stories are
+    stored in different files; the summary appears at the end of the story as
+    sentences that are prefixed by the special `@highlight` line. To process
+    the data, untar both datasets in the same folder, and pass the path to this
     folder as the "data_dir argument. The formatting code was inspired by [2].
 
     [1] https://cs.nyu.edu/~kcho/
     [2] https://github.com/abisee/cnn-dailymail/
     """
-    def __init_(self, tokenizer, data_dir='', block_size=512):
+
+    def __init_(self, tokenizer, data_dir="", block_size=512):
         assert os.path.isdir(data_dir)
 
         # Load features that have already been computed if present
-        cached_features_file = os.path.join(directory, "cached_lm_{}_{}".format(block_size, data_dir))
+        cached_features_file = os.path.join(
+            data_dir, "cached_lm_{}_{}".format(block_size, data_dir)
+        )
         if os.path.exists(cached_features_file):
             logger.info("Loading features from cached file %s", cached_features_file)
             with open(cached_features_file, "rb") as source:
@@ -78,7 +82,7 @@ class TextDataset(Dataset):
 
         logger.info("Creating features from dataset at %s", data_dir)
 
-        datasets = ['cnn', 'dailymail']
+        datasets = ["cnn", "dailymail"]
         for dataset in datasets:
             path_to_stories = os.path.join(data_dir, dataset, "stories")
             assert os.path.isdir(path_to_stories)
@@ -99,7 +103,9 @@ class TextDataset(Dataset):
                 story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
                 summary = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
                 story_seq, summary_seq = _fit_to_block_size(story, summary, block_size)
-                example = tokenizer.add_special_token_sequence_pair(story_seq, summary_seq)
+                example = tokenizer.add_special_token_sequence_pair(
+                    story_seq, summary_seq
+                )
                 self.examples.append(example)
 
         logger.info("Saving features into cache file %s", cached_features_file)
@@ -117,7 +123,9 @@ def process_story(raw_story):
     """ Process the text contained in a story file.
     Returns the story and the summary
     """
-    file_lines = list(filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")]))
+    file_lines = list(
+        filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")])
+    )
 
     # for some unknown reason some lines miss a period, add it
     file_lines = [_add_missing_period(line) for line in file_lines]
@@ -145,7 +153,7 @@ def process_story(raw_story):
 
 
 def _add_missing_period(line):
-    END_TOKENS = ['.', '!', '?', '...', "'", "`", '"', u'\u2019', u'\u2019', ")"]
+    END_TOKENS = [".", "!", "?", "...", "'", "`", '"', u"\u2019", u"\u2019", ")"]
     if line.startswith("@highlight"):
         return line
     if line[-1] in END_TOKENS:
@@ -154,34 +162,35 @@ def _add_missing_period(line):
 
 
 def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
-    """ Concatenate the sequences and adapt their lengths to the block size.
+    """ Adapt the source and target sequences' lengths to the block size.
 
-    Following [1] we truncate the source and target + tokens sequences so they fit
-    in the block size. If the concatenated sequence is longer than 512 we follow
-    the 75%/25% rule in [1]: limit the source sequence's length to 384 and the
-    target sequence's length to 128.
+    If the concatenated sequence (source + target + 3 special tokens) would be
+    longer than the block size we use the 75% / 25% rule followed in [1]. For a
+    block size of 512 this means limiting the source sequence's length to 384
+    and the target sequence's length to 128.
 
     [1] Dong, Li, et al. "Unified Language Model Pre-training for Natural
     Language Understanding and Generation." arXiv preprint arXiv:1905.03197 (2019).
     """
     SRC_MAX_LENGTH = int(0.75 * block_size) - 2  # CLS and EOS token
-    TGT_MAX_LENGTH = block_size - SRC_MAX_LENGTH - 1  # EOS token
+    TGT_MAX_LENGTH = block_size - (SRC_MAX_LENGTH + 2) - 1  # EOS token
 
-    # we dump the examples that are too small to fit in the block size for the
+    # We dump the examples that are too small to fit in the block size for the
     # sake of simplicity. You can modify this by adding model-specific padding.
-    if len(src_sequence) + len(src_sequence) + 3 < block_size:
+    if len(src_sequence) + len(tgt_sequence) + 3 < block_size:
         return None
 
-    # the source sequence has `[SEP_i]` special tokens with i \in [0,9]. We keep them for now.
     if len(src_sequence) > SRC_MAX_LENGTH:
         if len(tgt_sequence) > TGT_MAX_LENGTH:
             src_sequence = src_sequence[:SRC_MAX_LENGTH]
             tgt_sequence = tgt_sequence[:TGT_MAX_LENGTH]
         else:
-            src_sequence = src_sequence[block_size - len(tgt_sequence) - 3]
+            remain_size = block_size - len(tgt_sequence) - 3
+            src_sequence = src_sequence[:remain_size]
     else:
         if len(tgt_sequence) > TGT_MAX_LENGTH:
-            tgt_sequence = tgt_sequence[block_size - len(src_sequence) - 3]
+            remain_size = block_size - len(src_sequence) - 3
+            tgt_sequence = tgt_sequence[:remain_size]
 
     return src_sequence, tgt_sequence
 
@@ -200,44 +209,50 @@ def main():
     parser = argparse.ArgumentParser()
 
     # Required parameters
-    parser.add_argument("--data_dir",
-                        default=None,
-                        type=str,
-                        required=True,
-                        help="The input training data file (a text file).")
-    parser.add_argument("--output_dir",
-                        default=None,
-                        type=str,
-                        required=True,
-                        help="The output directory where the model predictions and checkpoints will be written.")
+    parser.add_argument(
+        "--data_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The input training data file (a text file).",
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
 
     # Optional parameters
-    parser.add_argument("--model_name_or_path",
-                        default="bert-base-cased",
-                        type=str,
-                        help="The model checkpoint for weights initialization.")
+    parser.add_argument(
+        "--model_name_or_path",
+        default="bert-base-cased",
+        type=str,
+        help="The model checkpoint for weights initialization.",
+    )
     parser.add_argument("--seed", default=42, type=int)
     args = parser.parse_args()
 
     # Set up training device
-    device = torch.device("cpu")
+    # device = torch.device("cpu")
 
     # Set seed
     set_seed(args)
 
     # Load pretrained model and tokenizer
-    config_class, model_class, tokenizer_class = BertConfig, Bert2Rnd, BertTokenizer
-    config = config_class.from_pretrained(args.model_name_or_path)
+    tokenizer_class = BertTokenizer
+    # config = config_class.from_pretrained(args.model_name_or_path)
     tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
-    model = model_class.from_pretrained(args.model_name_or_path, config=config)
-    model.to(device)
+    # model = model_class.from_pretrained(args.model_name_or_path, config=config)
+    # model.to(device)
 
     logger.info("Training/evaluation parameters %s", args)
 
     # Training
-    train_dataset = load_and_cache_examples(args, tokenizer)
-    global_step, tr_loss = train(args, train_dataset, model, tokenizer)
-    logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+    _ = load_and_cache_examples(args, tokenizer)
+    # global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+    # logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
 
 
 if __name__ == "__main__":
diff --git a/examples/run_seq2seq_finetuning_test.py b/examples/run_seq2seq_finetuning_test.py
index 34d9add10d..aff39f25b8 100644
--- a/examples/run_seq2seq_finetuning_test.py
+++ b/examples/run_seq2seq_finetuning_test.py
@@ -14,50 +14,50 @@
 # limitations under the License.
 import unittest
 
-from .run_seq2seq_finetuning import process_story, _fit_to_block_size
+from run_seq2seq_finetuning import _fit_to_block_size
 
 
 class DataLoaderTest(unittest.TestCase):
-    def __init__(self, block_size=10):
-        self.block_size = block_size
+    def setUp(self):
+        self.block_size = 10
 
-    def source_and_target_too_small(self):
+    def test_source_and_target_too_small(self):
         """ When the sum of the lengths of the source and target sequences is
         smaller than the block size (minus the number of special tokens), skip the example. """
         src_seq = [1, 2, 3, 4]
         tgt_seq = [5, 6]
         self.assertEqual(_fit_to_block_size(src_seq, tgt_seq, self.block_size), None)
 
-    def source_and_target_fit_exactly(self):
+    def test_source_and_target_fit_exactly(self):
         """ When the sum of the lengths of the source and target sequences is
         equal to the block size (minus the number of special tokens), return the
         sequences unchanged. """
         src_seq = [1, 2, 3, 4]
         tgt_seq = [5, 6, 7]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(src_seq == fitted_src)
-        self.assertListEqual(tgt_seq == fitted_tgt)
+        self.assertListEqual(src_seq, fitted_src)
+        self.assertListEqual(tgt_seq, fitted_tgt)
 
-    def source_too_big_target_ok(self):
+    def test_source_too_big_target_ok(self):
         src_seq = [1, 2, 3, 4, 5, 6]
         tgt_seq = [1, 2]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(src_seq == [1, 2, 3, 4, 5])
-        self.assertListEqual(tgt_seq == fitted_tgt)
+        self.assertListEqual(fitted_src, [1, 2, 3, 4, 5])
+        self.assertListEqual(fitted_tgt, fitted_tgt)
 
-    def target_too_big_source_ok(self):
+    def test_target_too_big_source_ok(self):
         src_seq = [1, 2, 3, 4]
         tgt_seq = [1, 2, 3, 4]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(src_seq == src_seq)
-        self.assertListEqual(tgt_seq == [1, 2, 3])
+        self.assertListEqual(fitted_src, src_seq)
+        self.assertListEqual(fitted_tgt, [1, 2, 3])
 
-    def source_and_target_too_big(self):
+    def test_source_and_target_too_big(self):
         src_seq = [1, 2, 3, 4, 5, 6, 7]
         tgt_seq = [1, 2, 3, 4, 5, 6, 7]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(src_seq == [1, 2, 3, 4, 5])
-        self.assertListEqual(tgt_seq == [1, 2])
+        self.assertListEqual(fitted_src, [1, 2, 3, 4, 5])
+        self.assertListEqual(fitted_tgt, [1, 2])
 
 
 if __name__ == "__main__":

From 1aec940587255083b2451fc18aa604de29c1188c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 15:18:07 +0200
Subject: [PATCH 043/269] test the full story processing

---
 examples/run_seq2seq_finetuning.py      | 32 +++++++++++------
 examples/run_seq2seq_finetuning_test.py | 46 +++++++++++++++++++++----
 2 files changed, 62 insertions(+), 16 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index e926523a17..f05a5847ed 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -87,9 +87,9 @@ class TextDataset(Dataset):
             path_to_stories = os.path.join(data_dir, dataset, "stories")
             assert os.path.isdir(path_to_stories)
 
-            stories_files = os.listdir(path_to_stories)
-            for story_file in stories_files:
-                path_to_story = os.path.join(path_to_stories, "story_file")
+            story_filenames_list = os.listdir(path_to_stories)
+            for story_filename in story_filenames_list:
+                path_to_story = os.path.join(path_to_stories, story_filename)
                 if not os.path.isfile(path_to_story):
                     continue
 
@@ -97,16 +97,16 @@ class TextDataset(Dataset):
                     try:
                         raw_story = source.read()
                         story, summary = process_story(raw_story)
-                    except IndexError:
+                    except IndexError:  # skip ill-formed stories
                         continue
 
                 story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
                 summary = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
                 story_seq, summary_seq = _fit_to_block_size(story, summary, block_size)
-                example = tokenizer.add_special_token_sequence_pair(
-                    story_seq, summary_seq
+
+                self.examples.append(
+                    tokenizer.add_special_token_sequence_pair(story_seq, summary_seq)
                 )
-                self.examples.append(example)
 
         logger.info("Saving features into cache file %s", cached_features_file)
         with open(cached_features_file, "wb") as sink:
@@ -120,8 +120,13 @@ class TextDataset(Dataset):
 
 
 def process_story(raw_story):
-    """ Process the text contained in a story file.
-    Returns the story and the summary
+    """ Extract the story and summary from a story file.
+
+    Attributes:
+        raw_story (str): content of the story file as an utf-8 encoded string.
+
+    Raises:
+        IndexError: If the stoy is empty or contains no highlights.
     """
     file_lines = list(
         filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")])
@@ -158,7 +163,7 @@ def _add_missing_period(line):
         return line
     if line[-1] in END_TOKENS:
         return line
-    return line + " ."
+    return line + "."
 
 
 def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
@@ -169,6 +174,13 @@ def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
     block size of 512 this means limiting the source sequence's length to 384
     and the target sequence's length to 128.
 
+    Attributes:
+        src_sequence (list): a list of ids that maps to the tokens of the
+            source sequence.
+        tgt_sequence (list): a list of ids that maps to the tokens of the
+            target sequence.
+        block_size (int): the model's block size.
+
     [1] Dong, Li, et al. "Unified Language Model Pre-training for Natural
     Language Understanding and Generation." arXiv preprint arXiv:1905.03197 (2019).
     """
diff --git a/examples/run_seq2seq_finetuning_test.py b/examples/run_seq2seq_finetuning_test.py
index aff39f25b8..e59f016da4 100644
--- a/examples/run_seq2seq_finetuning_test.py
+++ b/examples/run_seq2seq_finetuning_test.py
@@ -14,21 +14,21 @@
 # limitations under the License.
 import unittest
 
-from run_seq2seq_finetuning import _fit_to_block_size
+from run_seq2seq_finetuning import _fit_to_block_size, process_story
 
 
 class DataLoaderTest(unittest.TestCase):
     def setUp(self):
         self.block_size = 10
 
-    def test_source_and_target_too_small(self):
+    def test_truncate_source_and_target_too_small(self):
         """ When the sum of the lengths of the source and target sequences is
         smaller than the block size (minus the number of special tokens), skip the example. """
         src_seq = [1, 2, 3, 4]
         tgt_seq = [5, 6]
         self.assertEqual(_fit_to_block_size(src_seq, tgt_seq, self.block_size), None)
 
-    def test_source_and_target_fit_exactly(self):
+    def test_truncate_source_and_target_fit_exactly(self):
         """ When the sum of the lengths of the source and target sequences is
         equal to the block size (minus the number of special tokens), return the
         sequences unchanged. """
@@ -38,27 +38,61 @@ class DataLoaderTest(unittest.TestCase):
         self.assertListEqual(src_seq, fitted_src)
         self.assertListEqual(tgt_seq, fitted_tgt)
 
-    def test_source_too_big_target_ok(self):
+    def test_truncate_source_too_big_target_ok(self):
         src_seq = [1, 2, 3, 4, 5, 6]
         tgt_seq = [1, 2]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
         self.assertListEqual(fitted_src, [1, 2, 3, 4, 5])
         self.assertListEqual(fitted_tgt, fitted_tgt)
 
-    def test_target_too_big_source_ok(self):
+    def test_truncate_target_too_big_source_ok(self):
         src_seq = [1, 2, 3, 4]
         tgt_seq = [1, 2, 3, 4]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
         self.assertListEqual(fitted_src, src_seq)
         self.assertListEqual(fitted_tgt, [1, 2, 3])
 
-    def test_source_and_target_too_big(self):
+    def test_truncate_source_and_target_too_big(self):
         src_seq = [1, 2, 3, 4, 5, 6, 7]
         tgt_seq = [1, 2, 3, 4, 5, 6, 7]
         fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
         self.assertListEqual(fitted_src, [1, 2, 3, 4, 5])
         self.assertListEqual(fitted_tgt, [1, 2])
 
+    def test_process_story_no_highlights(self):
+        """ Processing a story with no highlights should raise an exception.
+        """
+        raw_story = """It was the year of Our Lord one thousand seven hundred and
+        seventy-five.\n\nSpiritual revelations were conceded to England at that
+        favoured period, as at this."""
+        with self.assertRaises(IndexError):
+            process_story(raw_story)
+
+    def test_process_empty_story(self):
+        """ An empty story should also raise and exception.
+        """
+        raw_story = ""
+        with self.assertRaises(IndexError):
+            process_story(raw_story)
+
+    def test_story_with_missing_period(self):
+        raw_story = (
+            "It was the year of Our Lord one thousand seven hundred and "
+            "seventy-five\n\nSpiritual revelations were conceded to England "
+            "at that favoured period, as at this.\n@highlight\n\nIt was the best of times"
+        )
+        story, summary = process_story(raw_story)
+
+        expected_story = (
+            "It was the year of Our Lord one thousand seven hundred and "
+            "seventy-five. Spiritual revelations were conceded to England at that "
+            "favoured period, as at this."
+        )
+        self.assertEqual(expected_story, story)
+
+        expected_summary = "It was the best of times."
+        self.assertEqual(expected_summary, summary)
+
 
 if __name__ == "__main__":
     unittest.main()

From 19e99647806ef597e2b21fd6ec2fed6624bdb696 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 15:20:28 +0200
Subject: [PATCH 044/269] remove Bert2Bert from module declaration

---
 transformers/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 006ba9ed16..5248bc9f1b 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -64,7 +64,7 @@ if is_torch_available():
                                 BertForMaskedLM, BertForNextSentencePrediction,
                                 BertForSequenceClassification, BertForMultipleChoice,
                                 BertForTokenClassification, BertForQuestionAnswering,
-                                load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, Bert2Rnd)
+                                load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel,
                                 OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel,
                                 load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP)

From 0d81fc853edac730067c0a2b3120dcc87ca6d15e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 15:26:33 +0200
Subject: [PATCH 045/269] specify in readme that both datasets are required

---
 examples/README.md | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index ba58a61012..e0fe1fc704 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -395,13 +395,17 @@ This fine-tuned model is available as a checkpoint under the reference
 
 Based on the script [`run_seq2seq_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_seq2seq_finetuning.py).
 
-Before running this script you should download **both** CNN and Daily Mail datasets (the links next to "Stories") from [Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/) in the same folder. Then uncompress the archives by running:
+Before running this script you should download **both** CNN and Daily Mail
+datasets from [Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/)  (the
+links next to "Stories") in the same folder. Then uncompress the archives by running:
 
 ```bash
 tar -xvf cnn_stories.tgz && tar -xvf dailymail_stories.tgz
 ```
 
-We will refer as `$DATA_PATH` the path to where you uncompressed both archive.
+note that the finetuning script **will not work** if you do not download both
+datasets. We will refer as `$DATA_PATH` the path to where you uncompressed both
+archive.
 
 ## Bert2Bert and abstractive summarization
 
@@ -414,4 +418,4 @@ python run_seq2seq_finetuning.py \
     --model_name_or_path=bert2bert \
     --do_train \
     --data_path=$DATA_PATH \
-```
\ No newline at end of file
+```

From 6d6c32673726896d682f71a40476576972d127b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 16:07:07 +0200
Subject: [PATCH 046/269] take path to pretrained for encoder and decoder for
 init

---
 transformers/modeling_seq2seq.py | 61 ++++++++++++++------------------
 1 file changed, 27 insertions(+), 34 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 466a101f47..2154a4699d 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -21,21 +21,20 @@ import logging
 import torch
 from torch import nn
 
-from .modeling_auto import AutoModel, AutoModelWithLMHead
-
-from .modeling_utils import PreTrainedModel, SequenceSummary
-
 from .file_utils import add_start_docstrings
+from .modeling_auto import AutoModel, AutoModelWithLMHead
+from .modeling_utils import PreTrainedModel, SequenceSummary
 
 logger = logging.getLogger(__name__)
 
 
 class PreTrainedSeq2seq(nn.Module):
     r"""
-        :class:`~transformers.Seq2seq` is a generic model class
-        that will be instantiated as a Seq2seq model with one of the base model classes of the library
-        as encoder and (optionally) as decoder when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)`
-        class method.
+        :class:`~transformers.Seq2seq` is a generic model class that will be
+        instantiated as a Seq2seq model with one of the base model classes of
+        the library as encoder and (optionally) as decoder when created with
+        the `AutoModel.from_pretrained(pretrained_model_name_or_path)` class
+        method.
     """
     def __init__(self, encoder, decoder):
         super(PreTrainedSeq2seq, self).__init__()
@@ -43,7 +42,7 @@ class PreTrainedSeq2seq(nn.Module):
         self.decoder = decoder
 
     @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+    def from_pretrained(cls, encoder_pretrained_model_name_or_path, decoder_pretrained_model_name_or_path, *model_args, **kwargs):
         r""" Instantiates one of the base model classes of the library
         from a pre-trained model configuration.
             The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
@@ -100,40 +99,34 @@ class PreTrainedSeq2seq(nn.Module):
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
             model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
-
         """
-        # Extract encoder and decoder model if provided
-        encoder_model = kwargs.pop('encoder_model', None)
-        decoder_model = kwargs.pop('decoder_model', None)
 
-        # Extract decoder kwargs so we only have encoder kwargs for now
-        if decoder_model is None:
-            decoder_pretrained_model_name_or_path = kwargs.pop('decoder_pretrained_model_name_or_path', pretrained_model_name_or_path)
-        decoder_kwargs = {}
-        for key in kwargs.keys():
+        # Separate the encoder- and decoder- specific kwargs. A kwarg is
+        # decoder-specific it the key starts with `decoder_`
+        kwargs_decoder = {}
+        kwargs_encoder = kwargs
+        for key in kwargs_encoder.keys():
             if key.startswith('decoder_'):
-                decoder_kwargs[key.replace('decoder_', '')] = kwargs.pop(key)
+                kwargs_decoder[key.replace('decoder_', '')] = kwargs_encoder.pop(key)
 
-        # Load and initialize the decoder
-        if encoder_model:
-            encoder = encoder_model
-        else:
-            # Load and initialize the encoder
-            kwargs['is_decoder'] = False  # Make sure the encoder will be an encoder
-            encoder = AutoModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        # Load and initialize the encoder and decoder
+        #  The distinction between encoder and decoder at the model level is made
+        #  by the value of the flag `is_decoder` that we need to set correctly.
+        encoder = kwargs.pop('encoder_model', None)
+        if encoder is None:
+            kwargs_encoder['is_decoder'] = False
+            encoder = AutoModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs_encoder)
 
-        # Load and initialize the decoder
-        if decoder_model:
-            decoder = decoder_model
-        else:
-            kwargs.update(decoder_kwargs)  # Replace encoder kwargs with decoder specific kwargs like config, state_dict, etc...
-            kwargs['is_decoder'] = True  # Make sure the decoder will be a decoder
-            decoder = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+        decoder = kwargs.pop('decoder_model', None)
+        if decoder is None:
+            kwargs_decoder['is_decoder'] = True
+            decoder_model = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
 
         model = cls(encoder, decoder)
+
         return model
 
-    def forward(self, *inputs, *kwargs):
+    def forward(self, *inputs, **kwargs):
         # Extract decoder inputs
         decoder_kwargs = {}
         for key in kwargs.keys():

From 4c81960b9bc0f553ddf800df16bb82804e162bcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 17:53:38 +0200
Subject: [PATCH 047/269] comment the seq2seq functions

---
 transformers/modeling_seq2seq.py | 81 +++++++++++++++++++-------------
 1 file changed, 49 insertions(+), 32 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 2154a4699d..b326f2bc1e 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -43,13 +43,21 @@ class PreTrainedSeq2seq(nn.Module):
 
     @classmethod
     def from_pretrained(cls, encoder_pretrained_model_name_or_path, decoder_pretrained_model_name_or_path, *model_args, **kwargs):
-        r""" Instantiates one of the base model classes of the library
-        from a pre-trained model configuration.
-            The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
-            To train the model, you should first set it back in training mode with `model.train()`
+        r""" Instantiates an encoder and a decoder from one or two base classes
+        of the library from pre-trained model checkpoints.
+
+
+        The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
+        To train the model, you need to first set it back in training mode with `model.train()`
 
         Params:
-            pretrained_model_name_or_path: either:
+            encoder_pretrained_model_name_or_path: information necessary to initiate the encoder. Either:
+
+                - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
+                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
+                - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+
+            decoder_pretrained_model_name_or_path: information necessary to initiate the decoder. Either:
 
                 - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
                 - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
@@ -84,21 +92,17 @@ class PreTrainedSeq2seq(nn.Module):
             output_loading_info: (`optional`) boolean:
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
 
-            kwargs: (`optional`) Remaining dictionary of keyword arguments:
+            kwargs: (`optional`) Remaining dictionary of keyword arguments.
                 Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
 
                 - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
                 - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
 
+                You can specify different kwargs for the decoder by prefixing the key with `decoder_` (e.g. ``decoder_output_attention=True``).
+
         Examples::
 
-            model = AutoModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
-            model = AutoModel.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModel.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
-            assert model.config.output_attention == True
-            # Loading from a TF checkpoint file instead of a PyTorch model (slower)
-            config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
-            model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
+            model = PreTrainedSeq2seq.from_pretained('bert-base-uncased', 'bert-base-uncased') # initialize Bert2Bert
         """
 
         # Separate the encoder- and decoder- specific kwargs. A kwarg is
@@ -115,35 +119,49 @@ class PreTrainedSeq2seq(nn.Module):
         encoder = kwargs.pop('encoder_model', None)
         if encoder is None:
             kwargs_encoder['is_decoder'] = False
-            encoder = AutoModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs_encoder)
+            encoder = AutoModel.from_pretrained(encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder)
 
         decoder = kwargs.pop('decoder_model', None)
         if decoder is None:
             kwargs_decoder['is_decoder'] = True
-            decoder_model = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs)
+            decoder = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)
 
         model = cls(encoder, decoder)
 
         return model
 
     def forward(self, *inputs, **kwargs):
-        # Extract decoder inputs
-        decoder_kwargs = {}
-        for key in kwargs.keys():
-            if key.startswith('decoder_'):
-                decoder_kwargs[key.replace('decoder_', '')] = kwargs.pop(key)
+        """ The forward pass on a seq2eq depends what we are performing:
 
-        # Compute encoder hidden states if needed
-        encoder_hidden_states = kwargs.pop('encoder_hidden_states', None)
+        - During training we perform one forward pass through both the encoder
+          and decoder;
+        - During prediction, we perform one forward pass through the encoder,
+          and then perform several forward passes with the encoder's hidden
+          state through the decoder to decode a full sequence.
+
+        Therefore, we skip the forward pass on the encoder if an argument named
+        `encoder_hidden_state` is passed to this function.
+
+        """
+        # Separate the encoder- and decoder- specific kwargs. A kwarg is
+        # decoder-specific it the key starts with `decoder_`
+        kwargs_decoder = {}
+        kwargs_encoder = kwargs
+        for key in kwargs_encoder.keys():
+            if key.startswith('decoder_'):
+                kwargs_decoder[key.replace('decoder_', '')] = kwargs_encoder.pop(key)
+
+        # Encode if needed (training, first prediction pass)
+        encoder_hidden_states = kwargs_encoder.pop('encoder_hidden_states', None)
         if encoder_hidden_states is None:
-            encoder_outputs = self.encoder(*inputs, *kwargs)
+            encoder_outputs = self.encoder(*inputs, **kwargs_encoder)
             encoder_hidden_states = encoder_outputs[0]
         else:
-            encoder_outputs = (,)
+            encoder_outputs = ()
 
         # Decode
-        decoder_kwargs['encoder_hidden_states'] = encoder_hidden_states
-        decoder_outputs = self.decoder(**decoder_kwargs)
+        kwargs_decoder['encoder_hidden_states'] = encoder_hidden_states
+        decoder_outputs = self.decoder(**kwargs_decoder)
 
         return decoder_outputs + encoder_outputs
 
@@ -161,11 +179,10 @@ class Model2LSTM(PreTrainedSeq2seq):
             # We will create a randomly initilized LSTM model as decoder
             if 'decoder_config' not in kwargs:
                 raise ValueError("To load an LSTM in Seq2seq model, please supply either: "
-                                "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or "
-                                "    - a dictionary of configuration parameters that will be used to initialize a
-                                "        torch.nn.LSTM model as `decoder_config` keyword argument. "
-                                "        E.g. `decoder_config=\{'input_size': 768, 'hidden_size': 768, 'num_layers': 2\}`")
-            kwargs['decoder_model'] = torch.nn.LSTM(kwarg.pop('decoder_config'))
+                                 "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or"
+                                 "    - a dictionary of configuration parameters that will be used to initialize a"
+                                 "      torch.nn.LSTM model as `decoder_config` keyword argument. "
+                                 "      E.g. `decoder_config={'input_size': 768, 'hidden_size': 768, 'num_layers': 2}`")
+            kwargs['decoder_model'] = torch.nn.LSTM(kwargs.pop('decoder_config'))
         model = super(Model2LSTM, cls).from_pretrained(*args, **kwargs)
         return model
-

From 488a6641513face9c03b537b6fe3210dbdb39f36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Tue, 15 Oct 2019 21:03:32 +0200
Subject: [PATCH 048/269] add `is_decoder` attribute to `PretrainedConfig`

We currenctly instantiate encoders and decoders for the seq2seq by
passing the `is_decoder` keyword argument to the `from_pretrained`
classmethod. On the other hand, the model class looks for the value
of the `is_decoder` attribute in its config.

In order for the value to propagate from the kwarg to the configuration
we simply need to define `is_decoder` as an attribute to the base
`PretrainedConfig`, with a default at `False`.
---
 transformers/configuration_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py
index 8a23be4ff6..228150fc89 100644
--- a/transformers/configuration_utils.py
+++ b/transformers/configuration_utils.py
@@ -56,6 +56,7 @@ class PretrainedConfig(object):
         self.torchscript = kwargs.pop('torchscript', False)
         self.use_bfloat16 = kwargs.pop('use_bfloat16', False)
         self.pruned_heads = kwargs.pop('pruned_heads', {})
+        self.is_decoder = kwargs.pop('is_decoder', False)
 
     def save_pretrained(self, save_directory):
         """ Save a configuration object to the directory `save_directory`, so that it

From c5a94a6100afdd550fb3ea445d8bddc6b9769fcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Wed, 16 Oct 2019 12:50:36 +0200
Subject: [PATCH 049/269] fix function that defines masks in XLM

the definition of `get_masks` would blow with the proper combination of
arguments. It was just a matter of moving a definition outside of a
control structure.
---
 transformers/modeling_xlm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py
index b29e721556..f1df6f668f 100644
--- a/transformers/modeling_xlm.py
+++ b/transformers/modeling_xlm.py
@@ -73,16 +73,16 @@ def get_masks(slen, lengths, causal, padding_mask=None):
     """
     Generate hidden states mask, and optionally an attention mask.
     """
-    bs = lengths.size(0)
+    alen = torch.arange(slen, dtype=torch.long, device=lengths.device)
     if padding_mask is not None:
         mask = padding_mask
     else:
         assert lengths.max().item() <= slen
-        alen = torch.arange(slen, dtype=torch.long, device=lengths.device)
         mask = alen < lengths[:, None]
 
     # attention mask is the same as mask, or triangular inferior attention (causal)
     if causal:
+        bs = lengths.size(0)
         attn_mask = alen[None, None, :].repeat(bs, slen, 1) <= alen[None, :, None]
     else:
         attn_mask = mask

From 075206961700fd2359f5c5cfc86a8c18d8404406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remi.louf@kering.com>
Date: Wed, 16 Oct 2019 16:12:22 +0200
Subject: [PATCH 050/269] adapt attention masks for the decoder case

The introduction of a decoder introduces 2 changes:
- We need to be able to specify a separate mask in the cross
attention to mask the positions corresponding to padding tokens in the
encoder state.
- The self-attention in the decoder needs to be causal on top of not
attending to padding tokens.
---
 transformers/modeling_bert.py | 66 +++++++++++++++++++++++++++--------
 1 file changed, 51 insertions(+), 15 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index fbf3c84646..cd9151cf62 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -198,12 +198,16 @@ class BertSelfAttention(nn.Module):
         x = x.view(*new_x_shape)
         return x.permute(0, 2, 1, 3)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
         mixed_query_layer = self.query(hidden_states)
+
         # if the attention Module is a encoder-decoder self attention module
+        # they keys & values are given by the encoder; the attention mask
+        # needs to be such that there is no atention on the encoder's padding tokens.
         if encoder_hidden_states is not None:
             mixed_key_layer = self.key(encoder_hidden_states)
             mixed_value_layer = self.value(encoder_hidden_states)
+            attention_mask = encoder_attention_mask
         else:
             mixed_key_layer = self.key(hidden_states)
             mixed_value_layer = self.value(hidden_states)
@@ -284,8 +288,8 @@ class BertAttention(nn.Module):
         self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
-        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_states)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
+        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)
         attention_output = self.output(self_outputs[0], hidden_states)
         outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
         return outputs
@@ -330,13 +334,13 @@ class BertLayer(nn.Module):
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None):
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None, encoder_attention_mask=None):
         self_attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
         attention_output = self_attention_outputs[0]
         outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
 
         if self.is_decoder and encoder_hidden_state is not None:
-            cross_attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state)
+            cross_attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state, encoder_attention_mask)
             attention_output = cross_attention_outputs[0]
             outputs = outputs + cross_attention_outputs[1:]  # add cross attentions if we output attention weights
 
@@ -346,6 +350,7 @@ class BertLayer(nn.Module):
         return outputs
 
 
+# NOTE I think we may need to call encoder_hidden_states[i] for each layer
 class BertEncoder(nn.Module):
     def __init__(self, config):
         super(BertEncoder, self).__init__()
@@ -353,14 +358,14 @@ class BertEncoder(nn.Module):
         self.output_hidden_states = config.output_hidden_states
         self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None):
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
         all_hidden_states = ()
         all_attentions = ()
         for i, layer_module in enumerate(self.layer):
             if self.output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
-            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_states)
+            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_states, encoder_attention_mask)
             hidden_states = layer_outputs[0]
 
             if self.output_attentions:
@@ -579,6 +584,7 @@ class BertModel(BertPreTrainedModel):
     """
     def __init__(self, config):
         super(BertModel, self).__init__(config)
+        self.config = config
 
         self.embeddings = BertEmbeddings(config)
         self.encoder = BertEncoder(config)
@@ -601,18 +607,47 @@ class BertModel(BertPreTrainedModel):
             self.encoder.layer[layer].attention.prune_heads(heads)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None,
-                head_mask=None, encoder_hidden_state=None):
+                head_mask=None, encoder_hidden_state=None, encoder_attention_mask=None):
+        """ Forward pass on the Model.
+
+        The model can behave as an encoder (with only self-attention) as well
+        as a decoder, in which case a layer of cross-attention is added between
+        ever self-attention layer, following the architecture described in [1].
+
+        To behave like as a decoder the model needs to be initialized with the
+        `is_decoder` argument of the config set to `True`. An
+        `encoder_hidden_state` is expected as an input to the forward pass.
+        When a decoder, there are two kinds of attention masks to specify:
+
+        (1) Self-attention masks that need to be causal (only attends to
+        previous tokens);
+        (2) A cross-attention mask that prevents the module
+        from attending to the encoder' padding tokens.
+
+        [1] Vaswani, Ashish, et al. "Attention is all you need." Advances in
+        neural information processing systems. 2017.
+        """
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
         if token_type_ids is None:
             token_type_ids = torch.zeros_like(input_ids)
 
-        # We create a 3D attention mask from a 2D tensor mask.
-        # Sizes are [batch_size, 1, 1, to_seq_length]
-        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
-        # this attention mask is more simple than the triangular masking of causal attention
-        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
-        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+        # we may want to provide a mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just make it broadcastable to all heads.
+        if attention_mask.dims() == 3:
+            extended_attention_mask = attention_mask[:, None, :, :]
+
+        # provided a padding mask of dimensions [batch_size, seq_length]
+        # - if encoder, make it broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        # - if decoder, make it causal
+        if attention_mask.dims() == 2:
+            if self.config.is_decoder:
+                batch_size, seq_length = input_ids.size()
+                seq_ids = torch.arange(seq_length)
+                causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
+                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[None, None, :, :]
+            else:
+                extended_attention_mask = attention_mask[:, None, None, :]
 
         # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
         # masked positions, this operation will create a tensor which is 0.0 for
@@ -641,7 +676,8 @@ class BertModel(BertPreTrainedModel):
         encoder_outputs = self.encoder(embedding_output,
                                        attention_mask=extended_attention_mask,
                                        head_mask=head_mask,
-                                       encoder_hidden_state=encoder_hidden_state)
+                                       encoder_hidden_state=encoder_hidden_state,
+                                       encoder_attention_mask=encoder_attention_mask)
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 

From 33c01368b19701bc6e5ea886f108663752d31d86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 16 Oct 2019 18:13:05 +0200
Subject: [PATCH 051/269] remove Bert2Rnd test

---
 transformers/tests/modeling_bert_test.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index e649cd8ce8..6c39c4e4db 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -29,7 +29,7 @@ if is_torch_available():
     from transformers import (BertConfig, BertModel, BertForMaskedLM,
                               BertForNextSentencePrediction, BertForPreTraining,
                               BertForQuestionAnswering, BertForSequenceClassification,
-                              BertForTokenClassification, BertForMultipleChoice, Bert2Rnd)
+                              BertForTokenClassification, BertForMultipleChoice)
     from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -255,17 +255,6 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.num_choices])
             self.check_loss_output(result)
 
-        def create_and_check_bert2rnd(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
-            config.num_choices = self.num_choices
-            model = Bert2Rnd(config=config)
-            model.eval()
-            bert2rnd_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-            bert2rnd_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-            bert2rnd_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-            _ = model(bert2rnd_inputs_ids,
-                      attention_mask=bert2rnd_input_mask,
-                      token_type_ids=bert2rnd_token_type_ids)
-
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
             (config, input_ids, token_type_ids, input_mask,

From a424892fab8ddfe631d7498bc44072aa3a42eb3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 16 Oct 2019 18:24:32 +0200
Subject: [PATCH 052/269] correct syntax error: dim() and not dims()

---
 transformers/modeling_bert.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index cd9151cf62..e717031dcb 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -634,13 +634,13 @@ class BertModel(BertPreTrainedModel):
 
         # we may want to provide a mask of dimensions [batch_size, from_seq_length, to_seq_length]
         # ourselves in which case we just make it broadcastable to all heads.
-        if attention_mask.dims() == 3:
+        if attention_mask.dim() == 3:
             extended_attention_mask = attention_mask[:, None, :, :]
 
         # provided a padding mask of dimensions [batch_size, seq_length]
         # - if encoder, make it broadcastable to [batch_size, num_heads, seq_length, seq_length]
         # - if decoder, make it causal
-        if attention_mask.dims() == 2:
+        if attention_mask.dim() == 2:
             if self.config.is_decoder:
                 batch_size, seq_length = input_ids.size()
                 seq_ids = torch.arange(seq_length)
@@ -816,13 +816,15 @@ class BertForMaskedLM(BertPreTrainedModel):
                                    self.bert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None):
+                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            encoder_hidden_states=encoder_hidden_states,
+                            encoder_attention_mask=encoder_attention_mask)
 
         sequence_output = outputs[0]
         prediction_scores = self.cls(sequence_output)
@@ -833,6 +835,15 @@ class BertForMaskedLM(BertPreTrainedModel):
             masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
             outputs = (masked_lm_loss,) + outputs
 
+        if encoder_hidden_states is not None:
+            loss_fct = CrossEntropyLoss(ignore_index=-1)
+
+            # shift predictions scores and input ids by one before computing loss
+            prediction_scores = prediction_scores[:, :-1, :]
+            input_ids = input_ids[:, 1:, :]
+            seq2seq_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), input_ids.view(-1))
+            outputs = (seq2seq_loss,) + outputs
+
         return outputs  # (masked_lm_loss), prediction_scores, (hidden_states), (attentions)
 
 

From e4e0ee14bd481fe32e82578665284ea5bf4f5677 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 16 Oct 2019 20:05:32 +0200
Subject: [PATCH 053/269] add separator between data import and train

---
 examples/run_seq2seq_finetuning.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index f05a5847ed..2e8d0aa250 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -52,6 +52,10 @@ def set_seed(args):
     torch.manual_seed(args.seed)
 
 
+# ------------
+# Load dataset
+# ------------
+
 class TextDataset(Dataset):
     """ Abstracts the dataset used to train seq2seq models.
 
@@ -212,6 +216,11 @@ def load_and_cache_examples(args, tokenizer):
     return dataset
 
 
+# ------------
+# Train
+# ------------
+
+
 def train(args, train_dataset, model, tokenizer):
     """ Fine-tune the pretrained model on the corpus. """
     raise NotImplementedError

From 95ec1d08bef7b780653f9f2c59fddb4a97873cff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 16 Oct 2019 20:55:42 +0200
Subject: [PATCH 054/269] separate inputs into encoder & decoder inputs

---
 transformers/modeling_seq2seq.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index b326f2bc1e..8f27224a56 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -130,7 +130,7 @@ class PreTrainedSeq2seq(nn.Module):
 
         return model
 
-    def forward(self, *inputs, **kwargs):
+    def forward(self, encoder_input_ids, decoder_input_ids, **kwargs):
         """ The forward pass on a seq2eq depends what we are performing:
 
         - During training we perform one forward pass through both the encoder
@@ -142,6 +142,11 @@ class PreTrainedSeq2seq(nn.Module):
         Therefore, we skip the forward pass on the encoder if an argument named
         `encoder_hidden_state` is passed to this function.
 
+        Params:
+            encoder_input_ids: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``
+                Indices of encoder input sequence tokens in the vocabulary.
+            decoder_input_ids: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``
+                Indices of decoder input sequence tokens in the vocabulary.
         """
         # Separate the encoder- and decoder- specific kwargs. A kwarg is
         # decoder-specific it the key starts with `decoder_`
@@ -154,14 +159,14 @@ class PreTrainedSeq2seq(nn.Module):
         # Encode if needed (training, first prediction pass)
         encoder_hidden_states = kwargs_encoder.pop('encoder_hidden_states', None)
         if encoder_hidden_states is None:
-            encoder_outputs = self.encoder(*inputs, **kwargs_encoder)
+            encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder)
             encoder_hidden_states = encoder_outputs[0]
         else:
             encoder_outputs = ()
 
         # Decode
         kwargs_decoder['encoder_hidden_states'] = encoder_hidden_states
-        decoder_outputs = self.decoder(**kwargs_decoder)
+        decoder_outputs = self.decoder(decoder_input_ids, **kwargs_decoder)
 
         return decoder_outputs + encoder_outputs
 

From 9b71fc9a18bbd49a699a338abe1891320c818108 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 16 Oct 2019 21:31:38 +0200
Subject: [PATCH 055/269] tying weights is going to be a clusterfuck

---
 transformers/modeling_seq2seq.py | 81 ++++++++++++++++++++++----------
 1 file changed, 56 insertions(+), 25 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 8f27224a56..4e76a1b8e7 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -28,7 +28,7 @@ from .modeling_utils import PreTrainedModel, SequenceSummary
 logger = logging.getLogger(__name__)
 
 
-class PreTrainedSeq2seq(nn.Module):
+class PreTrainedSeq2seq(PreTrainedModel):
     r"""
         :class:`~transformers.Seq2seq` is a generic model class that will be
         instantiated as a Seq2seq model with one of the base model classes of
@@ -36,13 +36,20 @@ class PreTrainedSeq2seq(nn.Module):
         the `AutoModel.from_pretrained(pretrained_model_name_or_path)` class
         method.
     """
+
     def __init__(self, encoder, decoder):
         super(PreTrainedSeq2seq, self).__init__()
         self.encoder = encoder
         self.decoder = decoder
 
     @classmethod
-    def from_pretrained(cls, encoder_pretrained_model_name_or_path, decoder_pretrained_model_name_or_path, *model_args, **kwargs):
+    def from_pretrained(
+        cls,
+        encoder_pretrained_model_name_or_path,
+        decoder_pretrained_model_name_or_path,
+        *model_args,
+        **kwargs
+    ):
         r""" Instantiates an encoder and a decoder from one or two base classes
         of the library from pre-trained model checkpoints.
 
@@ -110,21 +117,25 @@ class PreTrainedSeq2seq(nn.Module):
         kwargs_decoder = {}
         kwargs_encoder = kwargs
         for key in kwargs_encoder.keys():
-            if key.startswith('decoder_'):
-                kwargs_decoder[key.replace('decoder_', '')] = kwargs_encoder.pop(key)
+            if key.startswith("decoder_"):
+                kwargs_decoder[key.replace("decoder_", "")] = kwargs_encoder.pop(key)
 
         # Load and initialize the encoder and decoder
         #  The distinction between encoder and decoder at the model level is made
         #  by the value of the flag `is_decoder` that we need to set correctly.
-        encoder = kwargs.pop('encoder_model', None)
+        encoder = kwargs.pop("encoder_model", None)
         if encoder is None:
-            kwargs_encoder['is_decoder'] = False
-            encoder = AutoModel.from_pretrained(encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder)
+            kwargs_encoder["is_decoder"] = False
+            encoder = AutoModel.from_pretrained(
+                encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder
+            )
 
-        decoder = kwargs.pop('decoder_model', None)
+        decoder = kwargs.pop("decoder_model", None)
         if decoder is None:
-            kwargs_decoder['is_decoder'] = True
-            decoder = AutoModelWithLMHead.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)
+            kwargs_decoder["is_decoder"] = True
+            decoder = AutoModelWithLMHead.from_pretrained(
+                decoder_pretrained_model_name_or_path, **kwargs_decoder
+            )
 
         model = cls(encoder, decoder)
 
@@ -153,11 +164,11 @@ class PreTrainedSeq2seq(nn.Module):
         kwargs_decoder = {}
         kwargs_encoder = kwargs
         for key in kwargs_encoder.keys():
-            if key.startswith('decoder_'):
-                kwargs_decoder[key.replace('decoder_', '')] = kwargs_encoder.pop(key)
+            if key.startswith("decoder_"):
+                kwargs_decoder[key.replace("decoder_", "")] = kwargs_encoder.pop(key)
 
         # Encode if needed (training, first prediction pass)
-        encoder_hidden_states = kwargs_encoder.pop('encoder_hidden_states', None)
+        encoder_hidden_states = kwargs_encoder.pop("encoder_hidden_states", None)
         if encoder_hidden_states is None:
             encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder)
             encoder_hidden_states = encoder_outputs[0]
@@ -165,29 +176,49 @@ class PreTrainedSeq2seq(nn.Module):
             encoder_outputs = ()
 
         # Decode
-        kwargs_decoder['encoder_hidden_states'] = encoder_hidden_states
+        kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states
         decoder_outputs = self.decoder(decoder_input_ids, **kwargs_decoder)
 
         return decoder_outputs + encoder_outputs
 
 
 class Model2Model(PreTrainedSeq2seq):
-    def tie_weights():
-        # We should tie encoder and decoder embeddings if possible here
-        pass
+    def __init__(self):
+        super(Model2Model, self).__init__()
+        self.tie_weights()
+
+    def tie_weights(self):
+        """ Tying the encoder and decoders' embeddings together.
+
+        We need for each to get down to the embedding weights. However the
+        different model classes are inconsistent to that respect:
+        - BertModel: embeddings.word_embeddings
+        - RoBERTa: embeddings.word_embeddings
+        - XLMModel: embeddings
+        - GPT2: wte
+        - BertForMaskedLM: bert.embeddings.word_embeddings
+        - RobertaForMaskedLM: roberta.embeddings.word_embeddings
+
+        argument of the XEmbedding layer for each model, but it is "blocked"
+        by a model-specific keyword (bert, )...
+        """
+        # self._tie_or_clone_weights(self.encoder, self.decoder)
+        raise NotImplementedError
 
 
 class Model2LSTM(PreTrainedSeq2seq):
     @classmethod
     def from_pretrained(cls, *args, **kwargs):
-        if kwargs.get('decoder_model', None) is None:
+        if kwargs.get("decoder_model", None) is None:
             # We will create a randomly initilized LSTM model as decoder
-            if 'decoder_config' not in kwargs:
-                raise ValueError("To load an LSTM in Seq2seq model, please supply either: "
-                                 "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or"
-                                 "    - a dictionary of configuration parameters that will be used to initialize a"
-                                 "      torch.nn.LSTM model as `decoder_config` keyword argument. "
-                                 "      E.g. `decoder_config={'input_size': 768, 'hidden_size': 768, 'num_layers': 2}`")
-            kwargs['decoder_model'] = torch.nn.LSTM(kwargs.pop('decoder_config'))
+            if "decoder_config" not in kwargs:
+                raise ValueError(
+                    "To load an LSTM in Seq2seq model, please supply either: "
+                    "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or"
+                    "    - a dictionary of configuration parameters that will be used to initialize a"
+                    "      torch.nn.LSTM model as `decoder_config` keyword argument. "
+                    "      E.g. `decoder_config={'input_size': 768, 'hidden_size': 768, 'num_layers': 2}`"
+                )
+            kwargs["decoder_model"] = torch.nn.LSTM(kwargs.pop("decoder_config"))
         model = super(Model2LSTM, cls).from_pretrained(*args, **kwargs)
         return model

From 624a5644cc9585b19c90cb2a20e343f7ff326d82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 09:27:56 +0200
Subject: [PATCH 056/269] revert black formatting to conform with lib style

---
 transformers/modeling_seq2seq.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 4e76a1b8e7..cc5cc53bc3 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -43,13 +43,7 @@ class PreTrainedSeq2seq(PreTrainedModel):
         self.decoder = decoder
 
     @classmethod
-    def from_pretrained(
-        cls,
-        encoder_pretrained_model_name_or_path,
-        decoder_pretrained_model_name_or_path,
-        *model_args,
-        **kwargs
-    ):
+    def from_pretrained(cls, encoder_pretrained_model_name_or_path, decoder_pretrained_model_name_or_path, *model_args, **kwargs):
         r""" Instantiates an encoder and a decoder from one or two base classes
         of the library from pre-trained model checkpoints.
 
@@ -190,7 +184,7 @@ class Model2Model(PreTrainedSeq2seq):
     def tie_weights(self):
         """ Tying the encoder and decoders' embeddings together.
 
-        We need for each to get down to the embedding weights. However the
+       We need for each to get down to the embedding weights. However the
         different model classes are inconsistent to that respect:
         - BertModel: embeddings.word_embeddings
         - RoBERTa: embeddings.word_embeddings

From 4e0f24348fcc9902664951677ffc7c8cc171443d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 09:41:53 +0200
Subject: [PATCH 057/269] document the MLM modification + raise exception on
 MLM training with encoder-decoder

---
 transformers/modeling_bert.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index e717031dcb..2553bc0efb 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -830,21 +830,30 @@ class BertForMaskedLM(BertPreTrainedModel):
         prediction_scores = self.cls(sequence_output)
 
         outputs = (prediction_scores,) + outputs[2:]  # Add hidden states and attention if they are here
+
+        # Although this may seem awkward, BertForMaskedLM supports two scenarios:
+        # 1. If a tensor that contains the indices of masked labels is provided,
+        #    the cross-entropy is the MLM cross-entropy that measures the likelihood
+        #    of predictions for masked words.
+        # 2. If encoder hidden states are provided we are in a causal situation where we
+        #    try to predict the next word for each input in the encoder.
+        if masked_lm_labels is not None and encoder_hidden_states is not None:
+            raise AttributeError("Masked LM training with an encoder-decoder is not supported.")
+
         if masked_lm_labels is not None:
-            loss_fct = CrossEntropyLoss(ignore_index=-1)
+            loss_fct = CrossEntropyLoss(ignore_index=-1)  # -1 index = padding token
             masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
             outputs = (masked_lm_loss,) + outputs
 
         if encoder_hidden_states is not None:
-            loss_fct = CrossEntropyLoss(ignore_index=-1)
-
-            # shift predictions scores and input ids by one before computing loss
+            # we are doing next-token prediction; shift prediction scores and input ids by one
             prediction_scores = prediction_scores[:, :-1, :]
             input_ids = input_ids[:, 1:, :]
+            loss_fct = CrossEntropyLoss(ignore_index=-1)
             seq2seq_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), input_ids.view(-1))
             outputs = (seq2seq_loss,) + outputs
 
-        return outputs  # (masked_lm_loss), prediction_scores, (hidden_states), (attentions)
+        return outputs  # (mlm_or_seq2seq_loss), prediction_scores, (hidden_states), (attentions)
 
 
 @add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """,

From 638fe7f5a4f5c3bec5b39cee374f13b4675cdb18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 10:13:07 +0200
Subject: [PATCH 058/269] correct composition of padding and causal masks

---
 transformers/modeling_bert.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 2553bc0efb..05ab3395de 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -288,8 +288,8 @@ class BertAttention(nn.Module):
         self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
-        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None, encoder_attention_mask=None):
+        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_state, encoder_attention_mask)
         attention_output = self.output(self_outputs[0], hidden_states)
         outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
         return outputs
@@ -350,7 +350,6 @@ class BertLayer(nn.Module):
         return outputs
 
 
-# NOTE I think we may need to call encoder_hidden_states[i] for each layer
 class BertEncoder(nn.Module):
     def __init__(self, config):
         super(BertEncoder, self).__init__()
@@ -365,7 +364,8 @@ class BertEncoder(nn.Module):
             if self.output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
-            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_states, encoder_attention_mask)
+            encoder_hidden_state = encoder_hidden_states[i]
+            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_state, encoder_attention_mask)
             hidden_states = layer_outputs[0]
 
             if self.output_attentions:
@@ -607,22 +607,26 @@ class BertModel(BertPreTrainedModel):
             self.encoder.layer[layer].attention.prune_heads(heads)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None,
-                head_mask=None, encoder_hidden_state=None, encoder_attention_mask=None):
+                head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
         """ Forward pass on the Model.
 
+        The values of the attention matrix (shape [batch_size, seq_length])
+        should be 1.0 for the position we want to attend to and 0. for the ones
+        we do not want to attend to.
+
         The model can behave as an encoder (with only self-attention) as well
         as a decoder, in which case a layer of cross-attention is added between
         ever self-attention layer, following the architecture described in [1].
 
         To behave like as a decoder the model needs to be initialized with the
         `is_decoder` argument of the config set to `True`. An
-        `encoder_hidden_state` is expected as an input to the forward pass.
+        `encoder_hidden_states` is expected as an input to the forward pass.
         When a decoder, there are two kinds of attention masks to specify:
 
         (1) Self-attention masks that need to be causal (only attends to
         previous tokens);
         (2) A cross-attention mask that prevents the module
-        from attending to the encoder' padding tokens.
+        from attending to the encoder's padding tokens.
 
         [1] Vaswani, Ashish, et al. "Attention is all you need." Advances in
         neural information processing systems. 2017.
@@ -632,20 +636,20 @@ class BertModel(BertPreTrainedModel):
         if token_type_ids is None:
             token_type_ids = torch.zeros_like(input_ids)
 
-        # we may want to provide a mask of dimensions [batch_size, from_seq_length, to_seq_length]
-        # ourselves in which case we just make it broadcastable to all heads.
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
         if attention_mask.dim() == 3:
             extended_attention_mask = attention_mask[:, None, :, :]
 
-        # provided a padding mask of dimensions [batch_size, seq_length]
-        # - if encoder, make it broadcastable to [batch_size, num_heads, seq_length, seq_length]
-        # - if decoder, make it causal
+        # Provided a padding mask of dimensions [batch_size, seq_length]
+        # - if the model is a decoder, apply a causal mask in addition to the padding mask
+        # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
         if attention_mask.dim() == 2:
             if self.config.is_decoder:
                 batch_size, seq_length = input_ids.size()
                 seq_ids = torch.arange(seq_length)
                 causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
-                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[None, None, :, :]
+                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
             else:
                 extended_attention_mask = attention_mask[:, None, None, :]
 
@@ -676,7 +680,7 @@ class BertModel(BertPreTrainedModel):
         encoder_outputs = self.encoder(embedding_output,
                                        attention_mask=extended_attention_mask,
                                        head_mask=head_mask,
-                                       encoder_hidden_state=encoder_hidden_state,
+                                       encoder_hidden_states=encoder_hidden_states,
                                        encoder_attention_mask=encoder_attention_mask)
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)

From 87d60b6e19ee1c6d818e6cd5b7a3c4f56f5471ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 10:18:19 +0200
Subject: [PATCH 059/269] reword explanation of encoder_attention_mask

---
 transformers/modeling_bert.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 05ab3395de..be8ec5ba21 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -201,9 +201,9 @@ class BertSelfAttention(nn.Module):
     def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
         mixed_query_layer = self.query(hidden_states)
 
-        # if the attention Module is a encoder-decoder self attention module
-        # they keys & values are given by the encoder; the attention mask
-        # needs to be such that there is no atention on the encoder's padding tokens.
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
         if encoder_hidden_states is not None:
             mixed_key_layer = self.key(encoder_hidden_states)
             mixed_value_layer = self.value(encoder_hidden_states)

From c1bc709c3545fbafd7d7d9da01ba89d35aff6a79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 10:41:53 +0200
Subject: [PATCH 060/269] correct the truncation and padding of dataset

---
 examples/run_seq2seq_finetuning.py | 48 +++++++-----------------------
 1 file changed, 10 insertions(+), 38 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 2e8d0aa250..32f1782cab 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -104,9 +104,11 @@ class TextDataset(Dataset):
                     except IndexError:  # skip ill-formed stories
                         continue
 
-                story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
                 summary = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
-                story_seq, summary_seq = _fit_to_block_size(story, summary, block_size)
+                summary_seq = _fit_to_block_size(summary, block_size)
+
+                story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
+                story_seq = _fit_to_block_size(story, block_size)
 
                 self.examples.append(
                     tokenizer.add_special_token_sequence_pair(story_seq, summary_seq)
@@ -170,45 +172,15 @@ def _add_missing_period(line):
     return line + "."
 
 
-def _fit_to_block_size(src_sequence, tgt_sequence, block_size):
+def _fit_to_block_size(sequence, block_size):
     """ Adapt the source and target sequences' lengths to the block size.
-
-    If the concatenated sequence (source + target + 3 special tokens) would be
-    longer than the block size we use the 75% / 25% rule followed in [1]. For a
-    block size of 512 this means limiting the source sequence's length to 384
-    and the target sequence's length to 128.
-
-    Attributes:
-        src_sequence (list): a list of ids that maps to the tokens of the
-            source sequence.
-        tgt_sequence (list): a list of ids that maps to the tokens of the
-            target sequence.
-        block_size (int): the model's block size.
-
-    [1] Dong, Li, et al. "Unified Language Model Pre-training for Natural
-    Language Understanding and Generation." arXiv preprint arXiv:1905.03197 (2019).
+    If the sequence is shorter than the block size we pad it with -1 ids
+    which correspond to padding tokens.
     """
-    SRC_MAX_LENGTH = int(0.75 * block_size) - 2  # CLS and EOS token
-    TGT_MAX_LENGTH = block_size - (SRC_MAX_LENGTH + 2) - 1  # EOS token
-
-    # We dump the examples that are too small to fit in the block size for the
-    # sake of simplicity. You can modify this by adding model-specific padding.
-    if len(src_sequence) + len(tgt_sequence) + 3 < block_size:
-        return None
-
-    if len(src_sequence) > SRC_MAX_LENGTH:
-        if len(tgt_sequence) > TGT_MAX_LENGTH:
-            src_sequence = src_sequence[:SRC_MAX_LENGTH]
-            tgt_sequence = tgt_sequence[:TGT_MAX_LENGTH]
-        else:
-            remain_size = block_size - len(tgt_sequence) - 3
-            src_sequence = src_sequence[:remain_size]
+    if len(sequence) > block_size:
+        return sequence[:block_size]
     else:
-        if len(tgt_sequence) > TGT_MAX_LENGTH:
-            remain_size = block_size - len(src_sequence) - 3
-            tgt_sequence = tgt_sequence[:remain_size]
-
-    return src_sequence, tgt_sequence
+        return sequence.extend([-1] * [block_size - len(sequence)])
 
 
 def load_and_cache_examples(args, tokenizer):

From bfb9b540d408fd7f0592f321157fe0371c930c5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 12:59:51 +0200
Subject: [PATCH 061/269] add Model2Model to __init__

---
 examples/run_seq2seq_finetuning.py | 18 ++----------------
 transformers/__init__.py           |  1 +
 2 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 32f1782cab..94b29c3cd6 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -13,22 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Finetuning seq2seq models for sequence generation.
-
-We use the procedure described in [1] to finetune models for sequence
-generation. Let S1 and S2 be the source and target sequence respectively; we
-pack them using the start of sequence [EOS] and end of sequence [EOS] token:
-
-    [CLS] S1 [EOS] S2 [EOS]
-
-We then mask a fixed percentage of token from S2 at random and learn to predict
-the masked words. [EOS] can be masked during finetuning so the model learns to
-terminate the generation process.
-
-[1] Dong Li, Nan Yang, Wenhui Wang, Furu Wei, Xiaodong Liu, Yu Wang, Jianfeng
-Gao, Ming Zhou, and Hsiao-Wuen Hon.  “Unified Language Model Pre-Training for
-Natural Language Understanding and Generation.” (May 2019) ArXiv:1905.03197
-"""
+""" Finetuning seq2seq models for sequence generation."""
 
 import argparse
 from collections import deque
@@ -56,6 +41,7 @@ def set_seed(args):
 # Load dataset
 # ------------
 
+
 class TextDataset(Dataset):
     """ Abstracts the dataset used to train seq2seq models.
 
diff --git a/transformers/__init__.py b/transformers/__init__.py
index 5248bc9f1b..ee8e812a23 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -87,6 +87,7 @@ if is_torch_available():
     from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel,
                                 DistilBertForSequenceClassification, DistilBertForQuestionAnswering,
                                 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+    from .modeling_seq2seq import Model2Model
 
     # Optimization
     from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule,

From 47a06d88a00c59ea1fb54e92178b3f5d2e8e8973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 13:04:26 +0200
Subject: [PATCH 062/269] use two different tokenizers for storyand summary

---
 examples/run_seq2seq_finetuning.py | 54 ++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 94b29c3cd6..3e3cc34cb8 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -26,7 +26,7 @@ import numpy as np
 import torch
 from torch.utils.data import Dataset
 
-from transformers import BertTokenizer
+from transformers import AutoTokenizer, Model2Model
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +57,7 @@ class TextDataset(Dataset):
     [2] https://github.com/abisee/cnn-dailymail/
     """
 
-    def __init_(self, tokenizer, data_dir="", block_size=512):
+    def __init_(self, tokenizer_src, tokenizer_tgt, data_dir="", block_size=512):
         assert os.path.isdir(data_dir)
 
         # Load features that have already been computed if present
@@ -90,15 +90,13 @@ class TextDataset(Dataset):
                     except IndexError:  # skip ill-formed stories
                         continue
 
-                summary = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(summary))
-                summary_seq = _fit_to_block_size(summary, block_size)
-
-                story = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(story))
+                story = tokenizer_src.convert_tokens_to_ids(tokenizer_src.tokenize(story))
                 story_seq = _fit_to_block_size(story, block_size)
 
-                self.examples.append(
-                    tokenizer.add_special_token_sequence_pair(story_seq, summary_seq)
-                )
+                summary = tokenizer_tgt.convert_tokens_to_ids(tokenizer_tgt.tokenize(summary))
+                summary_seq = _fit_to_block_size(summary, block_size)
+
+                self.examples.append((story_seq, summary_seq))
 
         logger.info("Saving features into cache file %s", cached_features_file)
         with open(cached_features_file, "wb") as sink:
@@ -169,8 +167,8 @@ def _fit_to_block_size(sequence, block_size):
         return sequence.extend([-1] * [block_size - len(sequence)])
 
 
-def load_and_cache_examples(args, tokenizer):
-    dataset = TextDataset(tokenizer, file_path=args.data_dir)
+def load_and_cache_examples(args, tokenizer_src, tokenizer_tgt):
+    dataset = TextDataset(tokenizer_src, tokenizer_tgt, file_path=args.data_dir)
     return dataset
 
 
@@ -205,14 +203,35 @@ def main():
 
     # Optional parameters
     parser.add_argument(
-        "--model_name_or_path",
+        "--decoder_name_or_path",
         default="bert-base-cased",
         type=str,
-        help="The model checkpoint for weights initialization.",
+        help="The model checkpoint to initialize the decoder's weights with.",
+    )
+    parser.add_argument(
+        "--decoder_type",
+        default="bert",
+        type=str,
+        help="The decoder architecture to be fine-tuned.",
+    )
+    parser.add_argument(
+        "--encoder_name_or_path",
+        default="bert-base-cased",
+        type=str,
+        help="The model checkpoint to initialize the encoder's weights with.",
+    )
+    parser.add_argument(
+        "--encoder_type",
+        default="bert",
+        type=str,
+        help="The encoder architecture to be fine-tuned.",
     )
     parser.add_argument("--seed", default=42, type=int)
     args = parser.parse_args()
 
+    if args.encoder_type != 'bert' or args.decoder_type != 'bert':
+        raise ValueError("Only the BERT architecture is currently supported for seq2seq.")
+
     # Set up training device
     # device = torch.device("cpu")
 
@@ -220,16 +239,15 @@ def main():
     set_seed(args)
 
     # Load pretrained model and tokenizer
-    tokenizer_class = BertTokenizer
-    # config = config_class.from_pretrained(args.model_name_or_path)
-    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
-    # model = model_class.from_pretrained(args.model_name_or_path, config=config)
+    encoder_tokenizer_class = AutoTokenizer.from_pretrained(args.encoder_name_or_path)
+    decoder_tokenizer_class = AutoTokenizer.from_pretrained(args.decoder_name_or_path)
+    model = Model2Model.from_pretrained(args.encoder_name_or_path, args.decoder_name_or_path)
     # model.to(device)
 
     logger.info("Training/evaluation parameters %s", args)
 
     # Training
-    _ = load_and_cache_examples(args, tokenizer)
+    source, target = load_and_cache_examples(args, tokenizer)
     # global_step, tr_loss = train(args, train_dataset, model, tokenizer)
     # logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
 

From 578d23e06114bbd63cf5e931e0fdef9b8b6ac8c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 14:02:27 +0200
Subject: [PATCH 063/269] add training pipeline (formatting temporary)

---
 examples/run_seq2seq_finetuning.py | 139 +++++++++++++++++++++++++++--
 1 file changed, 130 insertions(+), 9 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 3e3cc34cb8..ad6d126165 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -23,8 +23,9 @@ import random
 import os
 
 import numpy as np
+from tqdm import tqdm, trange
 import torch
-from torch.utils.data import Dataset
+from torch.utils.data import Dataset, RandomSampler
 
 from transformers import AutoTokenizer, Model2Model
 
@@ -90,10 +91,14 @@ class TextDataset(Dataset):
                     except IndexError:  # skip ill-formed stories
                         continue
 
-                story = tokenizer_src.convert_tokens_to_ids(tokenizer_src.tokenize(story))
+                story = tokenizer_src.convert_tokens_to_ids(
+                    tokenizer_src.tokenize(story)
+                )
                 story_seq = _fit_to_block_size(story, block_size)
 
-                summary = tokenizer_tgt.convert_tokens_to_ids(tokenizer_tgt.tokenize(summary))
+                summary = tokenizer_tgt.convert_tokens_to_ids(
+                    tokenizer_tgt.tokenize(summary)
+                )
                 summary_seq = _fit_to_block_size(summary, block_size)
 
                 self.examples.append((story_seq, summary_seq))
@@ -179,7 +184,89 @@ def load_and_cache_examples(args, tokenizer_src, tokenizer_tgt):
 
 def train(args, train_dataset, model, tokenizer):
     """ Fine-tune the pretrained model on the corpus. """
-    raise NotImplementedError
+
+    # Prepare the data loading
+    args.train_bach_size = 1
+    train_sampler = RandomSampler(train_dataset)
+    train_dataloader = DataLoader(
+        train_dataset, sampler=train_sampler, batch_size=args.train_bach_size
+    )
+
+    # Prepare the optimizer and schedule (linear warmup and decay)
+    no_decay = ["bias", "LayerNorm.weight"]
+    optimizer_grouped_parameters = [
+        {
+            "params": [
+                p
+                for n, p in model.named_parameters()
+                if not any(nd in n for nd in no_decay)
+            ],
+            "weight_decay": args.weight_decay,
+        },
+        {
+            "params": [
+                p
+                for n, p in model.named_parameters()
+                if any(nd in n for nd in no_decay)
+            ],
+            "weight_decay": 0.0,
+        },
+    ]
+    optimizer = AdamW(
+        optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon
+    )
+    scheduler = WarmupLinearSchedule(
+        optimizer, warmup_steps=args.warmup_steps, t_total=t_total
+    )
+
+    # Train
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info(
+        "  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size
+    )
+    logger.info(
+        "  Total train batch size (w. parallel, distributed & accumulation) = %d",
+        args.train_batch_size
+        * args.gradient_accumulation_steps
+        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
+    )
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(args.num_train_epochs, desc="Epoch", disable=True)
+    set_seed(args)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=True)
+        for step, batch in enumerate(epoch_iterator):
+            source = ([s for s, _ in batch]).to(args.device)
+            target = ([t for _, t in batch]).to(args.device)
+            model.train()
+            outputs = model(source, target)
+            loss = outputs[0]
+            loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+                optimizer.step()
+                scheduler.step()
+                model.zero_grad()
+                global_step += 1
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    return global_step, tr_loss / global_step
 
 
 def main():
@@ -202,6 +289,9 @@ def main():
     )
 
     # Optional parameters
+    parser.add_argument(
+        "--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer."
+    )
     parser.add_argument(
         "--decoder_name_or_path",
         default="bert-base-cased",
@@ -226,11 +316,40 @@ def main():
         type=str,
         help="The encoder architecture to be fine-tuned.",
     )
+    parser.add_argument(
+        "--learning_rate",
+        default=5e-5,
+        type=float,
+        help="The initial learning rate for Adam.",
+    )
+    parser.add_argument(
+        "--max_grad_norm", default=1.0, type=float, help="Max gradient norm."
+    )
+    parser.add_argument(
+        "--max_steps",
+        default=-1,
+        type=int,
+        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
+    )
+    parser.add_argument(
+        "--num_train_epochs",
+        default=1,
+        type=int,
+        help="Total number of training epochs to perform.",
+    )
     parser.add_argument("--seed", default=42, type=int)
+    parser.add_argument(
+        "--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps."
+    )
+    parser.add_argument(
+        "--weight_decay", default=0.0, type=float, help="Weight deay if we apply some."
+    )
     args = parser.parse_args()
 
-    if args.encoder_type != 'bert' or args.decoder_type != 'bert':
-        raise ValueError("Only the BERT architecture is currently supported for seq2seq.")
+    if args.encoder_type != "bert" or args.decoder_type != "bert":
+        raise ValueError(
+            "Only the BERT architecture is currently supported for seq2seq."
+        )
 
     # Set up training device
     # device = torch.device("cpu")
@@ -241,14 +360,16 @@ def main():
     # Load pretrained model and tokenizer
     encoder_tokenizer_class = AutoTokenizer.from_pretrained(args.encoder_name_or_path)
     decoder_tokenizer_class = AutoTokenizer.from_pretrained(args.decoder_name_or_path)
-    model = Model2Model.from_pretrained(args.encoder_name_or_path, args.decoder_name_or_path)
+    model = Model2Model.from_pretrained(
+        args.encoder_name_or_path, args.decoder_name_or_path
+    )
     # model.to(device)
 
     logger.info("Training/evaluation parameters %s", args)
 
     # Training
-    source, target = load_and_cache_examples(args, tokenizer)
-    # global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+    train_dataset = load_and_cache_examples(args, tokenizer)
+    global_step, tr_loss = train(args, train_dataset, model, tokenizer)
     # logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
 
 

From 8cd56e30363fc00d947992ae412551f1775a5cfa Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Thu, 17 Oct 2019 16:33:26 +0200
Subject: [PATCH 064/269] fix data processing in script

---
 examples/run_seq2seq_finetuning.py | 49 +++++++++---------------------
 1 file changed, 15 insertions(+), 34 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index ad6d126165..38dcb2d005 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -58,12 +58,12 @@ class TextDataset(Dataset):
     [2] https://github.com/abisee/cnn-dailymail/
     """
 
-    def __init_(self, tokenizer_src, tokenizer_tgt, data_dir="", block_size=512):
+    def __init__(self, tokenizer, prefix='train', data_dir="", block_size=512):
         assert os.path.isdir(data_dir)
 
         # Load features that have already been computed if present
         cached_features_file = os.path.join(
-            data_dir, "cached_lm_{}_{}".format(block_size, data_dir)
+            data_dir, "cached_lm_{}_{}".format(block_size, prefix)
         )
         if os.path.exists(cached_features_file):
             logger.info("Loading features from cached file %s", cached_features_file)
@@ -72,7 +72,7 @@ class TextDataset(Dataset):
                 return
 
         logger.info("Creating features from dataset at %s", data_dir)
-
+        self.examples = []
         datasets = ["cnn", "dailymail"]
         for dataset in datasets:
             path_to_stories = os.path.join(data_dir, dataset, "stories")
@@ -91,21 +91,17 @@ class TextDataset(Dataset):
                     except IndexError:  # skip ill-formed stories
                         continue
 
-                story = tokenizer_src.convert_tokens_to_ids(
-                    tokenizer_src.tokenize(story)
-                )
+                story = tokenizer.encode(story)
                 story_seq = _fit_to_block_size(story, block_size)
 
-                summary = tokenizer_tgt.convert_tokens_to_ids(
-                    tokenizer_tgt.tokenize(summary)
-                )
+                summary = tokenizer.encode(summary)
                 summary_seq = _fit_to_block_size(summary, block_size)
 
                 self.examples.append((story_seq, summary_seq))
 
         logger.info("Saving features into cache file %s", cached_features_file)
         with open(cached_features_file, "wb") as sink:
-            pickle.dump(self.examples, sink, protocole=pickle.HIGHEST_PROTOCOL)
+            pickle.dump(self.examples, sink, protocol=pickle.HIGHEST_PROTOCOL)
 
     def __len__(self):
         return len(self.examples)
@@ -169,11 +165,11 @@ def _fit_to_block_size(sequence, block_size):
     if len(sequence) > block_size:
         return sequence[:block_size]
     else:
-        return sequence.extend([-1] * [block_size - len(sequence)])
+        return sequence.extend([-1] * (block_size - len(sequence)))
 
 
-def load_and_cache_examples(args, tokenizer_src, tokenizer_tgt):
-    dataset = TextDataset(tokenizer_src, tokenizer_tgt, file_path=args.data_dir)
+def load_and_cache_examples(args, tokenizer):
+    dataset = TextDataset(tokenizer, data_dir=args.data_dir)
     return dataset
 
 
@@ -293,29 +289,17 @@ def main():
         "--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer."
     )
     parser.add_argument(
-        "--decoder_name_or_path",
+        "--model_name_or_path",
         default="bert-base-cased",
         type=str,
-        help="The model checkpoint to initialize the decoder's weights with.",
+        help="The model checkpoint to initialize the encoder and decoder's weights with.",
     )
     parser.add_argument(
-        "--decoder_type",
+        "--model_type",
         default="bert",
         type=str,
         help="The decoder architecture to be fine-tuned.",
     )
-    parser.add_argument(
-        "--encoder_name_or_path",
-        default="bert-base-cased",
-        type=str,
-        help="The model checkpoint to initialize the encoder's weights with.",
-    )
-    parser.add_argument(
-        "--encoder_type",
-        default="bert",
-        type=str,
-        help="The encoder architecture to be fine-tuned.",
-    )
     parser.add_argument(
         "--learning_rate",
         default=5e-5,
@@ -346,7 +330,7 @@ def main():
     )
     args = parser.parse_args()
 
-    if args.encoder_type != "bert" or args.decoder_type != "bert":
+    if args.model_type != "bert":
         raise ValueError(
             "Only the BERT architecture is currently supported for seq2seq."
         )
@@ -358,11 +342,8 @@ def main():
     set_seed(args)
 
     # Load pretrained model and tokenizer
-    encoder_tokenizer_class = AutoTokenizer.from_pretrained(args.encoder_name_or_path)
-    decoder_tokenizer_class = AutoTokenizer.from_pretrained(args.decoder_name_or_path)
-    model = Model2Model.from_pretrained(
-        args.encoder_name_or_path, args.decoder_name_or_path
-    )
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
+    model = Model2Model.from_pretrained(args.model_name_or_path)
     # model.to(device)
 
     logger.info("Training/evaluation parameters %s", args)

From 56e2ee4eadc482a31ca46c97c3cc236824869510 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Thu, 17 Oct 2019 16:33:31 +0200
Subject: [PATCH 065/269] fix model2model

---
 transformers/modeling_seq2seq.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index cc5cc53bc3..ca3b9dc87a 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -28,7 +28,7 @@ from .modeling_utils import PreTrainedModel, SequenceSummary
 logger = logging.getLogger(__name__)
 
 
-class PreTrainedSeq2seq(PreTrainedModel):
+class PreTrainedSeq2seq(nn.Module):
     r"""
         :class:`~transformers.Seq2seq` is a generic model class that will be
         instantiated as a Seq2seq model with one of the base model classes of
@@ -43,7 +43,7 @@ class PreTrainedSeq2seq(PreTrainedModel):
         self.decoder = decoder
 
     @classmethod
-    def from_pretrained(cls, encoder_pretrained_model_name_or_path, decoder_pretrained_model_name_or_path, *model_args, **kwargs):
+    def from_pretrained(cls, encoder_pretrained_model_name_or_path=None, decoder_pretrained_model_name_or_path=None, *model_args, **kwargs):
         r""" Instantiates an encoder and a decoder from one or two base classes
         of the library from pre-trained model checkpoints.
 
@@ -177,8 +177,8 @@ class PreTrainedSeq2seq(PreTrainedModel):
 
 
 class Model2Model(PreTrainedSeq2seq):
-    def __init__(self):
-        super(Model2Model, self).__init__()
+    def __init__(self, *args, **kwargs):
+        super(Model2Model, self).__init__(*args, **kwargs)
         self.tie_weights()
 
     def tie_weights(self):
@@ -197,7 +197,14 @@ class Model2Model(PreTrainedSeq2seq):
         by a model-specific keyword (bert, )...
         """
         # self._tie_or_clone_weights(self.encoder, self.decoder)
-        raise NotImplementedError
+        pass
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        model = super(Model2Model, cls).from_pretrained(encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
+                                                        decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
+                                                        **kwargs)
+        return model
 
 
 class Model2LSTM(PreTrainedSeq2seq):

From 0919389d9aa03c19bee2ae9bc9922ff15ec8381b Mon Sep 17 00:00:00 2001
From: William Tambellini <wtambellini@sdl.com>
Date: Thu, 17 Oct 2019 14:41:04 -0700
Subject: [PATCH 066/269] Add speed log to examples/run_squad.py

Add a speed estimate log (time per example)
for evaluation to examples/run_squad.py
---
 examples/run_squad.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/examples/run_squad.py b/examples/run_squad.py
index 71c656a13d..f64ed13ae8 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -22,6 +22,7 @@ import logging
 import os
 import random
 import glob
+import timeit
 
 import numpy as np
 import torch
@@ -218,6 +219,7 @@ def evaluate(args, model, tokenizer, prefix=""):
     logger.info("  Num examples = %d", len(dataset))
     logger.info("  Batch size = %d", args.eval_batch_size)
     all_results = []
+    start_time = timeit.default_timer()
     for batch in tqdm(eval_dataloader, desc="Evaluating"):
         model.eval()
         batch = tuple(t.to(args.device) for t in batch)
@@ -250,6 +252,9 @@ def evaluate(args, model, tokenizer, prefix=""):
                                    end_logits   = to_list(outputs[1][i]))
             all_results.append(result)
 
+    evalTime = timeit.default_timer() - start_time
+    logger.info("  Evaluation done in total %f secs (%f sec per example)", evalTime, evalTime / len(dataset))
+
     # Compute predictions
     output_prediction_file = os.path.join(args.output_dir, "predictions_{}.json".format(prefix))
     output_nbest_file = os.path.join(args.output_dir, "nbest_predictions_{}.json".format(prefix))

From 3775550c4b27e29fac18a545ed87f84c7451aa61 Mon Sep 17 00:00:00 2001
From: Pasquale Minervini <p.minervini@gmail.com>
Date: Sun, 20 Oct 2019 22:33:56 +0100
Subject: [PATCH 067/269] gradient norm clipping should be done right before
 calling the optimiser

---
 examples/run_squad.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/examples/run_squad.py b/examples/run_squad.py
index 71c656a13d..aaf4952198 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -157,13 +157,16 @@ def train(args, train_dataset, model, tokenizer):
             if args.fp16:
                 with amp.scale_loss(loss, optimizer) as scaled_loss:
                     scaled_loss.backward()
-                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
             else:
                 loss.backward()
-                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
 
             tr_loss += loss.item()
             if (step + 1) % args.gradient_accumulation_steps == 0:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
                 optimizer.step()
                 scheduler.step()  # Update learning rate schedule
                 model.zero_grad()

From 4d456542e9d381090f9a00b2bcc5a4cb07f6f3f7 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 21 Oct 2019 16:34:14 +0200
Subject: [PATCH 068/269] Fix citation

---
 README.md | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index da0de4ae81..ad771f2ab1 100644
--- a/README.md
+++ b/README.md
@@ -549,12 +549,11 @@ for batch in train_data:
 
 We now have a paper you can cite for the 🤗 Transformers library:
 ```
-@misc{wolf2019transformers,
-    title={Transformers: State-of-the-art Natural Language Processing},
-    author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Jamie Brew},
-    year={2019},
-    eprint={1910.03771},
-    archivePrefix={arXiv},
-    primaryClass={cs.CL}
+@article{Wolf2019HuggingFacesTS,
+  title={HuggingFace's Transformers: State-of-the-art Natural Language Processing},
+  author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and R'emi Louf and Morgan Funtowicz and Jamie Brew},
+  journal={ArXiv},
+  year={2019},
+  volume={abs/1910.03771}
 }
 ```

From abd7110e21102467448035ffdbf6b208a05ac80b Mon Sep 17 00:00:00 2001
From: Pasquale Minervini <p.minervini@gmail.com>
Date: Mon, 21 Oct 2019 19:56:52 +0100
Subject: [PATCH 069/269] gradient norm clipping should be done right before
 calling the optimiser - fixing run_glue and run_ner as well

---
 examples/run_glue.py | 7 +++++--
 examples/run_ner.py  | 7 +++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/examples/run_glue.py b/examples/run_glue.py
index 45924c9290..54f6689e4d 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -154,13 +154,16 @@ def train(args, train_dataset, model, tokenizer):
             if args.fp16:
                 with amp.scale_loss(loss, optimizer) as scaled_loss:
                     scaled_loss.backward()
-                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
             else:
                 loss.backward()
-                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
 
             tr_loss += loss.item()
             if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
                 optimizer.step()
                 scheduler.step()  # Update learning rate schedule
                 model.zero_grad()
diff --git a/examples/run_ner.py b/examples/run_ner.py
index fdf2f1924a..00eb039258 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -133,13 +133,16 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id):
             if args.fp16:
                 with amp.scale_loss(loss, optimizer) as scaled_loss:
                     scaled_loss.backward()
-                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
             else:
                 loss.backward()
-                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
 
             tr_loss += loss.item()
             if (step + 1) % args.gradient_accumulation_steps == 0:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
                 scheduler.step()  # Update learning rate schedule
                 optimizer.step()
                 model.zero_grad()

From 777faa8ae7d9232b3b5ed1d6c7cb11dca3d744c3 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 11:26:42 -0400
Subject: [PATCH 070/269] Fix #1597

---
 transformers/tokenization_ctrl.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/transformers/tokenization_ctrl.py b/transformers/tokenization_ctrl.py
index 2406fa256b..c8d67ad043 100644
--- a/transformers/tokenization_ctrl.py
+++ b/transformers/tokenization_ctrl.py
@@ -63,11 +63,7 @@ def get_pairs(word):
 class CTRLTokenizer(PreTrainedTokenizer):
     """
     CTRL BPE tokenizer. Peculiarities:
-        - Byte-level Byte-Pair-Encoding
-        - Requires a space to start the input string => the encoding methods should be called with the
-          ``add_prefix_space`` flag set to ``True``.
-          Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve
-          the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"`
+        - Byte-Pair-Encoding
     """
     vocab_files_names = VOCAB_FILES_NAMES
     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP

From 1cfd9748683db43af2c98da1a19d39f0efc8cc3b Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 13:32:23 -0400
Subject: [PATCH 071/269] Option to benchmark only one of the two libraries

---
 examples/benchmarks.py | 55 ++++++++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/examples/benchmarks.py b/examples/benchmarks.py
index b1153bf566..d03844697d 100644
--- a/examples/benchmarks.py
+++ b/examples/benchmarks.py
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Benchmarking the library on inference and training """
-import tensorflow as tf
 
 # If checking the tensors placement
 # tf.debugging.set_log_device_placement(True)
@@ -23,15 +22,18 @@ from typing import List
 import timeit
 from transformers import is_tf_available, is_torch_available
 from time import time
-import torch
-
 import argparse
 import csv
 
-if not is_torch_available() or not is_tf_available():
-    raise ImportError("TensorFlow and Pytorch should be installed on the system.")
+if is_tf_available():
+    import tensorflow as tf
+    from transformers import TFAutoModel
 
-from transformers import AutoConfig, AutoModel, AutoTokenizer, TFAutoModel
+if is_torch_available():
+    import torch
+    from transformers import AutoModel
+
+from transformers import AutoConfig, AutoTokenizer
 
 input_text = """Bent over their instruments, three hundred Fertilizers were plunged, as 
 the Director of Hatcheries and Conditioning entered the room, in the 
@@ -434,26 +436,31 @@ def main():
     print("Running with arguments", args)
 
     if args.torch:
-        create_setup_and_compute(
-            model_names=args.models,
-            tensorflow=False,
-            gpu=args.torch_cuda,
-            torchscript=args.torchscript,
-            save_to_csv=args.save_to_csv,
-            csv_filename=args.csv_filename,
-            average_over=args.average_over
-        )
+        if is_torch_available():
+            create_setup_and_compute(
+                model_names=args.models,
+                tensorflow=False,
+                gpu=args.torch_cuda,
+                torchscript=args.torchscript,
+                save_to_csv=args.save_to_csv,
+                csv_filename=args.csv_filename,
+                average_over=args.average_over
+            )
+        else:
+            raise ImportError("Trying to run a PyTorch benchmark but PyTorch was not found in the environment.")
 
     if args.tensorflow:
-        create_setup_and_compute(
-            model_names=args.models,
-            tensorflow=True,
-            xla=args.xla,
-            save_to_csv=args.save_to_csv,
-            csv_filename=args.csv_filename,
-            average_over=args.average_over
-        )
-
+        if is_tf_available():
+            create_setup_and_compute(
+                model_names=args.models,
+                tensorflow=True,
+                xla=args.xla,
+                save_to_csv=args.save_to_csv,
+                csv_filename=args.csv_filename,
+                average_over=args.average_over
+            )
+        else:
+            raise ImportError("Trying to run a TensorFlow benchmark but TensorFlow was not found in the environment.")
 
 if __name__ == '__main__':
     main()

From 44286b94d3376f56ee7ef039790d40798d5f9e7d Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 13:46:48 -0400
Subject: [PATCH 072/269] RoBERTa doesn't print a warning when no special
 tokens are passed.

---
 transformers/modeling_roberta.py    | 12 ------------
 transformers/modeling_tf_roberta.py | 16 ----------------
 2 files changed, 28 deletions(-)

diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index eb340dc7fb..e15663d017 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -169,18 +169,6 @@ class RobertaModel(BertModel):
         self.embeddings = RobertaEmbeddings(config)
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
-        if input_ids[:, 0].sum().item() != 0:
-            logger.warning("A sequence with no special tokens has been passed to the RoBERTa model. "
-                           "This model requires special tokens in order to work. "
-                           "Please specify add_special_tokens=True in your tokenize.encode()"
-                           "or tokenizer.convert_tokens_to_ids().")
-        return super(RobertaModel, self).forward(input_ids,
-                                                 attention_mask=attention_mask,
-                                                 token_type_ids=token_type_ids,
-                                                 position_ids=position_ids,
-                                                 head_mask=head_mask)
-
 
 @add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """,
     ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py
index 244c83f2b3..b734f056ab 100644
--- a/transformers/modeling_tf_roberta.py
+++ b/transformers/modeling_tf_roberta.py
@@ -65,22 +65,6 @@ class TFRobertaMainLayer(TFBertMainLayer):
         super(TFRobertaMainLayer, self).__init__(config, **kwargs)
         self.embeddings = TFRobertaEmbeddings(config, name='embeddings')
 
-    def call(self, inputs, **kwargs):
-        # Check that input_ids starts with control token
-        if isinstance(inputs, (tuple, list)):
-            input_ids = inputs[0]
-        elif isinstance(inputs, dict):
-            input_ids = inputs.get('input_ids')
-        else:
-            input_ids = inputs
-
-        if tf.not_equal(tf.reduce_sum(input_ids[:, 0]), 0):
-            tf.print("A sequence with no special tokens has been passed to the RoBERTa model. "
-                           "This model requires special tokens in order to work. "
-                           "Please specify add_special_tokens=True in your encoding.")
-
-        return super(TFRobertaMainLayer, self).call(inputs, **kwargs)
-
 
 class TFRobertaPreTrainedModel(TFPreTrainedModel):
     """ An abstract class to handle weights initialization and

From 7d709e55ed54961ce3c84f53f1c14ee4f0c8a2e3 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 14:12:33 -0400
Subject: [PATCH 073/269] Remove

---
 examples/benchmarks.py                        |  4 +--
 .../distillation/scripts/binarized_data.py    |  2 +-
 examples/run_generation.py                    |  2 +-
 transformers/tests/tokenization_bert_test.py  |  4 +--
 .../tests/tokenization_distilbert_test.py     |  4 +--
 .../tests/tokenization_roberta_test.py        |  8 +++---
 .../tests/tokenization_tests_commons.py       | 28 ++++++++++---------
 transformers/tests/tokenization_xlm_test.py   |  4 +--
 transformers/tests/tokenization_xlnet_test.py |  4 +--
 transformers/tokenization_utils.py            | 20 ++++++-------
 10 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/examples/benchmarks.py b/examples/benchmarks.py
index d03844697d..06f368d946 100644
--- a/examples/benchmarks.py
+++ b/examples/benchmarks.py
@@ -309,7 +309,7 @@ def _compute_pytorch(model_names, dictionary, average_over, device, torchscript)
         model = AutoModel.from_pretrained(model_name, config=config)
         tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-        tokenized_sequence = tokenizer.encode(input_text)
+        tokenized_sequence = tokenizer.encode(input_text, add_special_tokens=False)
 
         max_input_size = tokenizer.max_model_input_sizes[model_name]
         batch_sizes = [1, 2, 4, 8]
@@ -353,7 +353,7 @@ def _compute_tensorflow(model_names, dictionary, average_over):
         model = TFAutoModel.from_pretrained(model_name, config=config)
         tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-        tokenized_sequence = tokenizer.encode(input_text)
+        tokenized_sequence = tokenizer.encode(input_text, add_special_tokens=False)
 
         max_input_size = tokenizer.max_model_input_sizes[model_name]
         batch_sizes = [1, 2, 4, 8]
diff --git a/examples/distillation/scripts/binarized_data.py b/examples/distillation/scripts/binarized_data.py
index 43824e9964..681cc2de34 100644
--- a/examples/distillation/scripts/binarized_data.py
+++ b/examples/distillation/scripts/binarized_data.py
@@ -68,7 +68,7 @@ def main():
     start = time.time()
     for text in data:
         text = f'{bos} {text.strip()} {sep}'
-        token_ids = tokenizer.encode(text)
+        token_ids = tokenizer.encode(text, add_special_tokens=False)
         rslt.append(token_ids)
 
         iter += 1
diff --git a/examples/run_generation.py b/examples/run_generation.py
index ef58cfd844..b7907e40da 100644
--- a/examples/run_generation.py
+++ b/examples/run_generation.py
@@ -223,7 +223,7 @@ def main():
         if args.model_type in ["transfo-xl", "xlnet"]:
             # Models with memory likes to have a long prompt for short inputs.
             raw_text = (args.padding_text if args.padding_text else PADDING_TEXT) + raw_text
-        context_tokens = tokenizer.encode(raw_text)
+        context_tokens = tokenizer.encode(raw_text, add_special_tokens=False)
         out = sample_sequence(
             model=model,
             context=context_tokens,
diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py
index 5e49e2915b..fd61ec30ba 100644
--- a/transformers/tests/tokenization_bert_test.py
+++ b/transformers/tests/tokenization_bert_test.py
@@ -128,8 +128,8 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester):
     def test_sequence_builders(self):
         tokenizer = self.tokenizer_class.from_pretrained("bert-base-uncased")
 
-        text = tokenizer.encode("sequence builders")
-        text_2 = tokenizer.encode("multi-sequence build")
+        text = tokenizer.encode("sequence builders", add_special_tokens=False)
+        text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
 
         encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
         encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
diff --git a/transformers/tests/tokenization_distilbert_test.py b/transformers/tests/tokenization_distilbert_test.py
index a18d644fe8..e3c8376ca8 100644
--- a/transformers/tests/tokenization_distilbert_test.py
+++ b/transformers/tests/tokenization_distilbert_test.py
@@ -33,8 +33,8 @@ class DistilBertTokenizationTest(BertTokenizationTest):
     def test_sequence_builders(self):
         tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
 
-        text = tokenizer.encode("sequence builders")
-        text_2 = tokenizer.encode("multi-sequence build")
+        text = tokenizer.encode("sequence builders", add_special_tokens=False)
+        text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
 
         encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
         encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
diff --git a/transformers/tests/tokenization_roberta_test.py b/transformers/tests/tokenization_roberta_test.py
index a731ac26c9..b31dd94f21 100644
--- a/transformers/tests/tokenization_roberta_test.py
+++ b/transformers/tests/tokenization_roberta_test.py
@@ -70,19 +70,19 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester):
         tokenizer = self.get_tokenizer()
 
         self.assertListEqual(
-            tokenizer.encode('Hello world!'),
+            tokenizer.encode('Hello world!', add_special_tokens=False),
             [0, 31414, 232, 328, 2]
         )
         self.assertListEqual(
-            tokenizer.encode('Hello world! cécé herlolip 418'),
+            tokenizer.encode('Hello world! cécé herlolip 418', add_special_tokens=False),
             [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]
         )
 
     def test_sequence_builders(self):
         tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
 
-        text = tokenizer.encode("sequence builders")
-        text_2 = tokenizer.encode("multi-sequence build")
+        text = tokenizer.encode("sequence builders", add_special_tokens=False)
+        text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
 
         encoded_text_from_decode = tokenizer.encode("sequence builders", add_special_tokens=True)
         encoded_pair_from_decode = tokenizer.encode("sequence builders", "multi-sequence build", add_special_tokens=True)
diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py
index b2801d5f41..a921696b77 100644
--- a/transformers/tests/tokenization_tests_commons.py
+++ b/transformers/tests/tokenization_tests_commons.py
@@ -79,13 +79,13 @@ class CommonTestCases:
             # Now let's start the test
             tokenizer = self.get_tokenizer(max_len=42)
 
-            before_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running")
+            before_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
 
             with TemporaryDirectory() as tmpdirname:
                 tokenizer.save_pretrained(tmpdirname)
                 tokenizer = self.tokenizer_class.from_pretrained(tmpdirname)
 
-                after_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running")
+                after_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
                 self.assertListEqual(before_tokens, after_tokens)
 
                 self.assertEqual(tokenizer.max_len, 42)
@@ -130,7 +130,7 @@ class CommonTestCases:
             self.assertEqual(added_toks, len(new_toks))
             self.assertEqual(all_size_2, all_size + len(new_toks))
 
-            tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l")
+            tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False)
             out_string = tokenizer.decode(tokens)
 
             self.assertGreaterEqual(len(tokens), 4)
@@ -148,7 +148,8 @@ class CommonTestCases:
             self.assertEqual(added_toks_2, len(new_toks_2))
             self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))
 
-            tokens = tokenizer.encode(">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l")
+            tokens = tokenizer.encode(">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l",
+                                      add_special_tokens=False)
             out_string = tokenizer.decode(tokens)
 
             self.assertGreaterEqual(len(tokens), 6)
@@ -166,7 +167,7 @@ class CommonTestCases:
 
             tokens = tokenizer.tokenize(input_text)
             ids = tokenizer.convert_tokens_to_ids(tokens)
-            ids_2 = tokenizer.encode(input_text)
+            ids_2 = tokenizer.encode(input_text, add_special_tokens=False)
             self.assertListEqual(ids, ids_2)
 
             tokens_2 = tokenizer.convert_ids_to_tokens(ids)
@@ -206,7 +207,7 @@ class CommonTestCases:
             seq_0 = "Test this method."
             seq_1 = "With these inputs."
 
-            sequences = tokenizer.encode(seq_0, seq_1)
+            sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=False)
             attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
 
             # Method is implemented (e.g. not GPT-2)
@@ -219,7 +220,7 @@ class CommonTestCases:
             seq_0 = "This is a sentence to be encoded."
             stride = 2
 
-            sequence = tokenizer.encode(seq_0)
+            sequence = tokenizer.encode(seq_0, add_special_tokens=False)
             num_added_tokens = tokenizer.num_added_tokens()
             total_length = len(sequence) + num_added_tokens
             information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride)
@@ -239,13 +240,13 @@ class CommonTestCases:
             seq_1 = "This is another sentence to be encoded."
             stride = 2
 
-            sequence_0_no_special_tokens = tokenizer.encode(seq_0)
-            sequence_1_no_special_tokens = tokenizer.encode(seq_1)
+            sequence_0_no_special_tokens = tokenizer.encode(seq_0, add_special_tokens=False)
+            sequence_1_no_special_tokens = tokenizer.encode(seq_1, add_special_tokens=False)
 
             sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
             truncated_second_sequence = tokenizer.build_inputs_with_special_tokens(
-                tokenizer.encode(seq_0),
-                tokenizer.encode(seq_1)[:-2]
+                tokenizer.encode(seq_0, add_special_tokens=False),
+                tokenizer.encode(seq_1, add_special_tokens=False)[:-2]
             )
 
             information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True,
@@ -283,7 +284,7 @@ class CommonTestCases:
             sequence_1 = "This one too please."
 
             # Testing single inputs
-            encoded_sequence = tokenizer.encode(sequence_0)
+            encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False)
             encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True)
             encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
             special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
@@ -294,7 +295,8 @@ class CommonTestCases:
             self.assertEqual(encoded_sequence, filtered_sequence)
 
             # Testing inputs pairs
-            encoded_sequence = tokenizer.encode(sequence_0) + tokenizer.encode(sequence_1)
+            encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) + tokenizer.encode(sequence_1,
+                                                                                                         add_special_tokens=False)
             encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True)
             encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
             special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
diff --git a/transformers/tests/tokenization_xlm_test.py b/transformers/tests/tokenization_xlm_test.py
index 0949b0cce4..567edf1ccd 100644
--- a/transformers/tests/tokenization_xlm_test.py
+++ b/transformers/tests/tokenization_xlm_test.py
@@ -69,8 +69,8 @@ class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester):
     def test_sequence_builders(self):
         tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048")
 
-        text = tokenizer.encode("sequence builders")
-        text_2 = tokenizer.encode("multi-sequence build")
+        text = tokenizer.encode("sequence builders", add_special_tokens=False)
+        text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
 
         encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
         encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
diff --git a/transformers/tests/tokenization_xlnet_test.py b/transformers/tests/tokenization_xlnet_test.py
index 1a5dbcf6df..653968b9af 100644
--- a/transformers/tests/tokenization_xlnet_test.py
+++ b/transformers/tests/tokenization_xlnet_test.py
@@ -92,8 +92,8 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester):
     def test_sequence_builders(self):
         tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
 
-        text = tokenizer.encode("sequence builders")
-        text_2 = tokenizer.encode("multi-sequence build")
+        text = tokenizer.encode("sequence builders", add_special_tokens=False)
+        text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
 
         encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
         encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py
index 5e5be872ef..ac765165e2 100644
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -689,14 +689,14 @@ class PreTrainedTokenizer(object):
         raise NotImplementedError
 
     def encode(self,
-                text,
-                text_pair=None,
-                add_special_tokens=False,
-                max_length=None,
-                stride=0,
-                truncation_strategy='longest_first',
-                return_tensors=None,
-                **kwargs):
+               text,
+               text_pair=None,
+               add_special_tokens=True,
+               max_length=None,
+               stride=0,
+               truncation_strategy='longest_first',
+               return_tensors=None,
+               **kwargs):
         """
         Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary.
 
@@ -739,7 +739,7 @@ class PreTrainedTokenizer(object):
     def encode_plus(self,
                     text,
                     text_pair=None,
-                    add_special_tokens=False,
+                    add_special_tokens=True,
                     max_length=None,
                     stride=0,
                     truncation_strategy='longest_first',
@@ -794,7 +794,7 @@ class PreTrainedTokenizer(object):
                                       truncation_strategy=truncation_strategy,
                                       return_tensors=return_tensors)
 
-    def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0,
+    def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0,
                           truncation_strategy='longest_first', return_tensors=None):
         """
         Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model.

From e16d46843a19ab289b82138e4eccec5610a76de7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juli=C3=A1n=20Peller=20=28dataista=29?=
 <julianpeller@gmail.com>
Date: Tue, 22 Oct 2019 16:11:02 -0300
Subject: [PATCH 074/269] Fix architectures count

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ad771f2ab1..e8506d6a39 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ State-of-the-art NLP for everyone
 Lower compute costs, smaller carbon footprint
 - Researchers can share trained models instead of always retraining
 - Practitioners can reduce compute time and production costs
-- 8 architectures with over 30 pretrained models, some in more than 100 languages
+- 10 architectures with over 30 pretrained models, some in more than 100 languages
 
 Choose the right framework for every part of a model's lifetime
 - Train state-of-the-art models in 3 lines of code
@@ -111,7 +111,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 
 ## Model architectures
 
-🤗 Transformers currently provides 8 NLU/NLG architectures:
+🤗 Transformers currently provides 10 NLU/NLG architectures:
 
 1. **[BERT](https://github.com/google-research/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
 2. **[GPT](https://github.com/openai/finetune-transformer-lm)** (from OpenAI) released with the paper [Improving Language Understanding by Generative Pre-Training](https://blog.openai.com/language-unsupervised/) by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever.

From ef1b8b2ae5ad1057154a126879f7eb8de685f862 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 22 Oct 2019 21:27:20 +0000
Subject: [PATCH 075/269] [CTRL] warn if generation prompt does not start with
 a control code

see also https://github.com/salesforce/ctrl/pull/50
---
 README.md                         |  2 +-
 examples/README.md                |  2 +-
 examples/run_generation.py        |  5 ++-
 transformers/tokenization_ctrl.py | 59 +++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e8506d6a39..ecba50a74e 100644
--- a/README.md
+++ b/README.md
@@ -413,7 +413,7 @@ and from the Salesforce CTRL model:
 python ./examples/run_generation.py \
     --model_type=ctrl \
     --length=20 \
-    --model_name_or_path=gpt2 \
+    --model_name_or_path=ctrl \
     --temperature=0 \
     --repetition_penalty=1.2 \
 ```
diff --git a/examples/README.md b/examples/README.md
index 6b68d880eb..3a76a4a830 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -101,7 +101,7 @@ python run_lm_finetuning.py \
 
 Based on the script [`run_generation.py`](https://github.com/huggingface/transformers/blob/master/examples/run_generation.py).
 
-Conditional text generation using the auto-regressive models of the library: GPT, GPT-2, Transformer-XL and XLNet.
+Conditional text generation using the auto-regressive models of the library: GPT, GPT-2, Transformer-XL, XLNet, CTRL.
 A similar script is used for our official demo [Write With Transfomer](https://transformer.huggingface.co), where you
 can try out the different models available in the library.
 
diff --git a/examples/run_generation.py b/examples/run_generation.py
index ef58cfd844..ae0e27dcf0 100644
--- a/examples/run_generation.py
+++ b/examples/run_generation.py
@@ -196,7 +196,7 @@ def main():
 
     logger.info(args)
     if args.model_type in ["ctrl"]:
-        if args.temperature > 0.7 : 
+        if args.temperature > 0.7:
             logger.info('CTRL typically works better with lower temperatures (and lower top_k).')
 
     while True:
@@ -224,6 +224,9 @@ def main():
             # Models with memory likes to have a long prompt for short inputs.
             raw_text = (args.padding_text if args.padding_text else PADDING_TEXT) + raw_text
         context_tokens = tokenizer.encode(raw_text)
+        if args.model_type == "ctrl":
+            if not any(context_tokens[0] == x for x in tokenizer.control_codes.values()):
+                logger.info("WARNING! You are not starting your generation from a control code so you won't get good results")
         out = sample_sequence(
             model=model,
             context=context_tokens,
diff --git a/transformers/tokenization_ctrl.py b/transformers/tokenization_ctrl.py
index c8d67ad043..3d67fa2c5b 100644
--- a/transformers/tokenization_ctrl.py
+++ b/transformers/tokenization_ctrl.py
@@ -46,6 +46,64 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
     'ctrl': 256,
 }
 
+CONTROL_CODES = {
+    "Pregnancy": 168629,
+    "Christianity": 7675,
+    "Explain": 106423,
+    "Fitness": 63440,
+    "Saving": 63163,
+    "Ask": 27171,
+    "Ass": 95985,
+    "Joke": 163509,
+    "Questions": 45622,
+    "Thoughts": 49605,
+    "Retail": 52342,
+    "Feminism": 164338,
+    "Writing": 11992,
+    "Atheism": 192263,
+    "Netflix": 48616,
+    "Computing": 39639,
+    "Opinion": 43213,
+    "Alone": 44967,
+    "Funny": 58917,
+    "Gaming": 40358,
+    "Human": 4088,
+    "India": 1331,
+    "Joker": 77138,
+    "Diet": 36206,
+    "Legal": 11859,
+    "Norman": 4939,
+    "Tip": 72689,
+    "Weight": 52343,
+    "Movies": 46273,
+    "Running": 23425,
+    "Science": 2090,
+    "Horror": 37793,
+    "Confession": 60572,
+    "Finance": 12250,
+    "Politics": 16360,
+    "Scary": 191985,
+    "Support": 12654,
+    "Technologies": 32516,
+    "Teenage": 66160,
+    "Event": 32769,
+    "Learned": 67460,
+    "Notion": 182770,
+    "Wikipedia": 37583,
+    "Books": 6665,
+    "Extract": 76050,
+    "Confessions": 102701,
+    "Conspiracy": 75932,
+    "Links": 63674,
+    "Narcissus": 150425,
+    "Relationship": 54766,
+    "Relationships": 134796,
+    "Reviews": 41671,
+    "News": 4256,
+    "Translation": 26820,
+    "multilingual": 128406,
+}
+
 def get_pairs(word):
     """Return set of symbol pairs in a word.
 
@@ -68,6 +126,7 @@ class CTRLTokenizer(PreTrainedTokenizer):
     vocab_files_names = VOCAB_FILES_NAMES
     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    control_codes = CONTROL_CODES
 
     def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
         super(CTRLTokenizer, self).__init__(unk_token=unk_token, **kwargs)

From bc3e57d551faa5f444107d0a41fcf56f4870ca8c Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 17:51:30 -0400
Subject: [PATCH 076/269] Multi version doc deployment

---
 .circleci/config.yml |  6 +++---
 .circleci/deploy.sh  | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100755 .circleci/deploy.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 637d137492..38d0e291af 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -81,12 +81,12 @@ jobs:
             - checkout
             - run: sudo pip install --progress-bar off -r docs/requirements.txt
             - run: sudo pip install --progress-bar off -r requirements.txt
-            - run: cd docs && make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir
+            - run: ./deploy.sh
 workflow_filters: &workflow_filters
     filters:
         branches:
             only:
-                - master
+                - deploy_doc
 workflows:
     version: 2
     build_and_test:
@@ -96,4 +96,4 @@ workflows:
             - build_py3_tf
             - build_py2_torch
             - build_py2_tf
-            - deploy_doc: *workflow_filters
\ No newline at end of file
+            - deploy_doc: *workflow_filters
diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
new file mode 100755
index 0000000000..19226ae4d5
--- /dev/null
+++ b/.circleci/deploy.sh
@@ -0,0 +1,21 @@
+cd docs
+
+function deploy_doc(){
+	echo "Creating doc at commit $1 and pushing to folder $2"
+	git checkout $1
+	if [ ! -z "$2" ] 
+	then
+		echo "Pushing version" $2
+		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+	else
+		echo "Pushing master"
+		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir
+	fi
+}
+
+deploy_doc "master" 
+deploy_doc "b33a385" v1.0.0
+deploy_doc "fe02e45" v1.1.0
+eploy_doc "89fd345" v1.2.0
+deploy_doc "fc9faa8" v2.0.0
+deploy_doc "3ddce1d" v2.1.1

From 69eba0ab19fda74ff00f1e4f5bded98a2fec9887 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 17:53:52 -0400
Subject: [PATCH 077/269] Edit script path

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 38d0e291af..4c6423ada4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -81,7 +81,7 @@ jobs:
             - checkout
             - run: sudo pip install --progress-bar off -r docs/requirements.txt
             - run: sudo pip install --progress-bar off -r requirements.txt
-            - run: ./deploy.sh
+            - run: ./.circleci/deploy.sh
 workflow_filters: &workflow_filters
     filters:
         branches:

From fbcc5ff9fbad6e42d5d852fe315eb4c01707ed2e Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 22 Oct 2019 18:01:10 -0400
Subject: [PATCH 078/269] Change branch to master

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4c6423ada4..01e6d82b33 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -86,7 +86,7 @@ workflow_filters: &workflow_filters
     filters:
         branches:
             only:
-                - deploy_doc
+                - master
 workflows:
     version: 2
     build_and_test:

From 6e85bccafc8a175c0cc2eed27fd61af087483085 Mon Sep 17 00:00:00 2001
From: Lysandre Debut <lysandre@huggingface.co>
Date: Tue, 22 Oct 2019 18:07:01 -0400
Subject: [PATCH 079/269] Fixed typo

---
 .circleci/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 19226ae4d5..2bff0102ae 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -16,6 +16,6 @@ function deploy_doc(){
 deploy_doc "master" 
 deploy_doc "b33a385" v1.0.0
 deploy_doc "fe02e45" v1.1.0
-eploy_doc "89fd345" v1.2.0
+deploy_doc "89fd345" v1.2.0
 deploy_doc "fc9faa8" v2.0.0
 deploy_doc "3ddce1d" v2.1.1

From bd847ce7d7a498c3852f6bb31af8f9e781a85f65 Mon Sep 17 00:00:00 2001
From: "focox@qq.com" <https://gitee.com/focox/respiration_detection.git>
Date: Wed, 23 Oct 2019 20:27:13 +0800
Subject: [PATCH 080/269] fixed the bug raised by "tmp_eval_loss +=
 tmp_eval_loss.item()" when parallelly using multi-gpu.

---
 examples/run_ner.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index 00eb039258..28d9e9db28 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -210,6 +210,9 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=""
             outputs = model(**inputs)
             tmp_eval_loss, logits = outputs[:2]
 
+            if args.n_gpu > 1:
+                tmp_eval_loss = tmp_eval_loss.mean()  # mean() to average on multi-gpu parallel evaluating
+
             eval_loss += tmp_eval_loss.item()
         nb_eval_steps += 1
         if preds is None:

From 8ad5c591cda96a40d2fd2662a6b76af86527289d Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Wed, 23 Oct 2019 10:29:47 -0400
Subject: [PATCH 081/269] [RELEASE] DistilRoBERTa

---
 docs/source/pretrained_models.rst |  4 ++++
 examples/distillation/README.md   | 38 +++++++++++++++++++++++--------
 2 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index 1d02cd0dd7..43c08228bd 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -136,6 +136,10 @@ Here is the full list of the currently provided pretrained models together with
 |                   | ``distilgpt2``                                             | | 6-layer, 768-hidden, 12-heads, 82M parameters                                                                                       |
 |                   |                                                            | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint.                                                               |
 |                   |                                                            | (see `details <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__)                                     |
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``distilroberta-base``                                     | | 6-layer, 768-hidden, 12-heads, 82M parameters                                                                                       |
+|                   |                                                            | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint.                                                 |
+|                   |                                                            | (see `details <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__)                                     |
 +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 | CTRL              | ``ctrl``                                                   | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters                                                                                    |
 |                   |                                                            | | Salesforce's Large-sized CTRL English model                                                                                         |
diff --git a/examples/distillation/README.md b/examples/distillation/README.md
index 0fbcb5628b..344b5f7d46 100644
--- a/examples/distillation/README.md
+++ b/examples/distillation/README.md
@@ -1,25 +1,38 @@
 # Distil*
 
-This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT and DistilGPT2.
+This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT, DistilRoBERTa and DistilGPT2.
 
-**2019, October 3rd - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Please use the paper as a reference when comparing/reporting results on DistilBERT.**
+**October 23rd, 2019 - Update** We release **DistilRoBERTa**: 95% of `RoBERTa-base`'s performance on GLUE, twice as fast as RoBERTa while being 35% smaller.
+
+**October 3rd, 2019 - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Please use the paper as a reference when comparing/reporting results on DistilBERT.**
+
+**September 19th, 2019 - Update:** We fixed bugs in the code and released an upadted version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 97% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future!
 
-**2019, September 19th - Update:** We fixed bugs in the code and released an upadted version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 97% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future!
 
 ## What is Distil*
 
 Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production.
 
-We have applied the same method to GPT2 and release the weights of the compressed model. On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 15.0 compared to 18.5 for DistilGPT2 (after fine-tuning on the train set).
+We have applied the same method to other Transformer architectures and released the weights:
+- GPT2: on the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 15.0 compared to 18.5 for **DistilGPT2** (after fine-tuning on the train set).
+- RoBERTa: **DistilRoBERTa** reaches 95% of `RoBERTa-base` performance on GLUE while being twice faster and 35% smaller.
+- and more to come! 🤗🤗🤗
 
 For more information on DistilBERT, please refer to our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108).
 
 Here are the results on the dev sets of GLUE:
 
-| Model      | Macro-score | CoLA | MNLI | MRPC | QNLI | QQP  | RTE  | SST-2| STS-B| WNLI |
-| :---:      |    :---:    | :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:|
-| BERT-base  |  **77.6**   | 48.9 | 84.3 | 88.6 | 89.3 | 89.5 | 71.3 | 91.7 | 91.2 | 43.7 |
-| DistilBERT |  **76.8**   | 49.1 | 81.8 | 90.2 | 90.2 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4 |
+| Model                     | Macro-score                    | CoLA | MNLI | MRPC | QNLI | QQP  | RTE  | SST-2| STS-B| WNLI              |
+| :---:                     |    :---:                       | :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:             |
+| BERT-base                 |  **77.6**                      | 48.9 | 84.3 | 88.6 | 89.3 | 89.5 | 71.3 | 91.7 | 91.2 | 43.7              |
+| DistilBERT                |  **76.8**                      | 49.1 | 81.8 | 90.2 | 90.2 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4              |
+| :---:                     |    :---:                       | :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:             |
+| RoBERTa-base (reported)   |  **83.2**/**86.4**<sup>2</sup> | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.7<sup>**</sup> |
+| DistilRoBERTa<sup>1</sup> |  **79.0**/**82.3**<sup>2</sup> | 59.4 | 83.9 | 86.6 | 90.8 | 89.4 | 67.9 | 92.5 | 88.3 | 52.1              |
+
+<sup>1</sup> We did not use the MNLI checkpoint for fine-tuning but directy perform transfer learning on the pre-trained DistilRoBERTa. 
+<sup>2</sup> Macro-score computed without WNLI.
+<sup>3</sup> We compute this score ourselves for completeness.
 
 ## Setup
 
@@ -27,13 +40,15 @@ This part of the library has only be tested with Python3.6+. There are few speci
 
 **Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breakings changes compared to v1.1.0).
 
+
 ## How to use DistilBERT
 
 Transformers includes two pre-trained Distil* models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT):
 
 - `distilbert-base-uncased`: DistilBERT English language model pretrained on the same data used to pretrain Bert (concatenation of the Toronto Book Corpus and full English Wikipedia) using distillation with the supervision of the `bert-base-uncased` version of Bert. The model has 6 layers, 768 dimension and 12 heads, totalizing 66M parameters.
 - `distilbert-base-uncased-distilled-squad`: A finetuned version of `distilbert-base-uncased` finetuned using (a second step of) knwoledge distillation on SQuAD 1.0. This model reaches a F1 score of 86.9 on the dev set (for comparison, Bert `bert-base-uncased` version reaches a 88.5 F1 score).
-- `distilgpt2`: DistilGPT2 English language model pretrained with the supervision of `gpt2` (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset and . The model has 6 layers, 768 dimension and 12 heads, totalizing 82M (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2.
+- `distilgpt2`: DistilGPT2 English language model pretrained with the supervision of `gpt2` (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset. The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2.
+- `distilroberta-base`: DistilRoBERTa English language model pretrained with the supervision of `roberta-base` solely on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset (it is ~4 times less training data than the teacher RoBERTa). The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 125M parameters for RoBERTa-base). On average DistilRoBERTa is twice as fast as Roberta-base.
 - and more to come! 🤗🤗🤗
 
 Using DistilBERT is very similar to using BERT. DistilBERT share the same tokenizer as BERT's `bert-base-uncased` even though we provide a link to this tokenizer under the `DistilBertTokenizer` name to have a consistent naming between the library models.
@@ -47,7 +62,10 @@ outputs = model(input_ids)
 last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
 ```
 
-Similarly, using DistilGPT2 simply consists in calling the GPT2 classes from a different pretrained checkpoint: `model = GPT2Model.from_pretrained('distilgpt2')`.
+Similarly, using the other Distil* models simply consists in calling the base classes with a different pretrained checkpoint:
+- DistilGPT2: `model = GPT2Model.from_pretrained('distilgpt2')`
+- DistilRoBERTa: `model = RobertaModel.from_pretrained('distilroberta-base')`
+
 
 ## How to train Distil*
 

From 5b6cafb11b39e78724dc13b57b81bd73c9a66b49 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Wed, 23 Oct 2019 10:35:16 -0400
Subject: [PATCH 082/269] [release] fix table weirdness

---
 examples/distillation/README.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/examples/distillation/README.md b/examples/distillation/README.md
index 344b5f7d46..7da1ad015b 100644
--- a/examples/distillation/README.md
+++ b/examples/distillation/README.md
@@ -26,12 +26,14 @@ Here are the results on the dev sets of GLUE:
 | :---:                     |    :---:                       | :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:             |
 | BERT-base                 |  **77.6**                      | 48.9 | 84.3 | 88.6 | 89.3 | 89.5 | 71.3 | 91.7 | 91.2 | 43.7              |
 | DistilBERT                |  **76.8**                      | 49.1 | 81.8 | 90.2 | 90.2 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4              |
-| :---:                     |    :---:                       | :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:             |
-| RoBERTa-base (reported)   |  **83.2**/**86.4**<sup>2</sup> | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.7<sup>**</sup> |
+| ---                       |    ---                         |  --- |  --- |  --- |  --- |  --- |  --- |  --- |  --- |  ---              |
+| RoBERTa-base (reported)   |  **83.2**/**86.4**<sup>2</sup> | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.7<sup>3</sup> |
 | DistilRoBERTa<sup>1</sup> |  **79.0**/**82.3**<sup>2</sup> | 59.4 | 83.9 | 86.6 | 90.8 | 89.4 | 67.9 | 92.5 | 88.3 | 52.1              |
 
-<sup>1</sup> We did not use the MNLI checkpoint for fine-tuning but directy perform transfer learning on the pre-trained DistilRoBERTa. 
+<sup>1</sup> We did not use the MNLI checkpoint for fine-tuning but directy perform transfer learning on the pre-trained DistilRoBERTa.
+
 <sup>2</sup> Macro-score computed without WNLI.
+
 <sup>3</sup> We compute this score ourselves for completeness.
 
 ## Setup

From b82bfbd0c307ba84da4f326900f1479df977efeb Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 24 Oct 2019 15:55:31 +0000
Subject: [PATCH 083/269] Updated README to show all available documentation

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e8506d6a39..3d89c65901 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime
 | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation |
 | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers |
 | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers |
-| [Documentation](https://huggingface.co/transformers/) | Full API documentation and more |
+| [Documentation](https://huggingface.co/transformers/) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more |
 
 ## Installation
 

From 66085a132161d3257bb971d886bea1b52a476e4e Mon Sep 17 00:00:00 2001
From: Matt Maybeno <mmaybeno@gmail.com>
Date: Wed, 23 Oct 2019 21:05:13 -0700
Subject: [PATCH 084/269] RoBERTa token classification

[WIP] copy paste bert token classification for roberta
---
 transformers/__init__.py                      |  2 +
 transformers/modeling_roberta.py              | 72 +++++++++++++++++++
 transformers/modeling_tf_roberta.py           | 51 +++++++++++++
 transformers/tests/modeling_roberta_test.py   | 19 ++++-
 .../tests/modeling_tf_roberta_test.py         | 15 ++++
 5 files changed, 158 insertions(+), 1 deletion(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index fbc92f078e..dbc66f86b9 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -89,6 +89,7 @@ if is_torch_available():
                             XLM_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_roberta import (RobertaForMaskedLM, RobertaModel,
                                 RobertaForSequenceClassification, RobertaForMultipleChoice,
+                                RobertaForTokenClassification,
                                 ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel,
                                 DistilBertForSequenceClassification, DistilBertForQuestionAnswering,
@@ -139,6 +140,7 @@ if is_tf_available():
     from .modeling_tf_roberta import (TFRobertaPreTrainedModel, TFRobertaMainLayer,
                                       TFRobertaModel, TFRobertaForMaskedLM,
                                       TFRobertaForSequenceClassification,
+                                      TFRobertaForTokenClassification,
                                       TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
 
     from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer,
diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index eb340dc7fb..6b8d381579 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -343,6 +343,7 @@ class RobertaForSequenceClassification(BertPreTrainedModel):
 
         return outputs  # (loss), logits, (hidden_states), (attentions)
 
+
 @add_start_docstrings("""Roberta Model with a multiple choice classification head on top (a linear layer on top of
     the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """,
     ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
@@ -451,6 +452,77 @@ class RobertaForMultipleChoice(BertPreTrainedModel):
         return outputs  # (loss), reshaped_logits, (hidden_states), (attentions)
 
 
+@add_start_docstrings("""Roberta Model with a token classification head on top (a linear layer on top of
+    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+    ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
+class RobertaForTokenClassification(BertPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the token classification loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification loss.
+        **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)``
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+        model = RobertaForTokenClassification.from_pretrained('roberta-base')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, scores = outputs[:2]
+
+    """
+    def __init__(self, config):
+        super(RobertaForTokenClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.roberta = RobertaModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, labels=None):
+
+        outputs = self.roberta(input_ids,
+                               attention_mask=attention_mask,
+                               token_type_ids=token_type_ids,
+                               position_ids=position_ids,
+                               head_mask=head_mask)
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output)
+        logits = self.classifier(sequence_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+            # Only keep active parts of the loss
+            if attention_mask is not None:
+                active_loss = attention_mask.view(-1) == 1
+                active_logits = logits.view(-1, self.num_labels)[active_loss]
+                active_labels = labels.view(-1)[active_loss]
+                loss = loss_fct(active_logits, active_labels)
+            else:
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), scores, (hidden_states), (attentions)
+
 
 class RobertaClassificationHead(nn.Module):
     """Head for sentence-level classification tasks."""
diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py
index 244c83f2b3..13a0522211 100644
--- a/transformers/modeling_tf_roberta.py
+++ b/transformers/modeling_tf_roberta.py
@@ -371,3 +371,54 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel):
         outputs = (logits,) + outputs[2:]
 
         return outputs  # logits, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""RoBERTa Model with a token classification head on top (a linear layer on top of
+    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+    ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
+class TFRobertaForTokenClassification(TFRobertaPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)``
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import RobertaTokenizer, TFRobertaForTokenClassification
+
+        tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+        model = TFRobertaForTokenClassification.from_pretrained('roberta-base')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        scores = outputs[0]
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFRobertaForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        self.num_labels = config.num_labels
+
+        self.roberta = TFRobertaMainLayer(config, name='roberta')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+        self.classifier = tf.keras.layers.Dense(config.num_labels,
+                                                kernel_initializer=get_initializer(config.initializer_range),
+                                                name='classifier')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.roberta(inputs, **kwargs)
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False))
+        logits = self.classifier(sequence_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        return outputs  # scores, (hidden_states), (attentions)
diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py
index 82e10da915..0620ddf630 100644
--- a/transformers/tests/modeling_roberta_test.py
+++ b/transformers/tests/modeling_roberta_test.py
@@ -24,7 +24,8 @@ from transformers import is_torch_available
 
 if is_torch_available():
     import torch
-    from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification)
+    from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM,
+                              RobertaForSequenceClassification, RobertaForTokenClassification)
     from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -156,6 +157,22 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.vocab_size])
             self.check_loss_output(result)
 
+        def create_and_check_roberta_for_token_classification(self, config, input_ids, token_type_ids, input_mask,
+                                                              sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = RobertaForTokenClassification(config=config)
+            model.eval()
+            loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids,
+                                 labels=token_labels)
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.seq_length, self.num_labels])
+            self.check_loss_output(result)
+
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
             (config, input_ids, token_type_ids, input_mask,
diff --git a/transformers/tests/modeling_tf_roberta_test.py b/transformers/tests/modeling_tf_roberta_test.py
index 735c9aae27..edbfa4e205 100644
--- a/transformers/tests/modeling_tf_roberta_test.py
+++ b/transformers/tests/modeling_tf_roberta_test.py
@@ -30,6 +30,7 @@ if is_tf_available():
     import numpy
     from transformers.modeling_tf_roberta import (TFRobertaModel, TFRobertaForMaskedLM,
                                                           TFRobertaForSequenceClassification,
+                                                          TFRobertaForTokenClassification,
                                                           TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
 else:
     pytestmark = pytest.mark.skip("Require TensorFlow")
@@ -154,6 +155,20 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester):
                 list(result["prediction_scores"].shape),
                 [self.batch_size, self.seq_length, self.vocab_size])
 
+        def create_and_check_roberta_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = TFRobertaForTokenClassification(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            logits, = model(inputs)
+            result = {
+                "logits": logits.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].shape),
+                [self.batch_size, self.seq_length, self.num_labels])
+
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
             (config, input_ids, token_type_ids, input_mask,

From b92d68421dee75c3a078b26b78a05bd59007d855 Mon Sep 17 00:00:00 2001
From: Matt Maybeno <mmaybeno@gmail.com>
Date: Wed, 23 Oct 2019 21:31:28 -0700
Subject: [PATCH 085/269] Use roberta model and update doc strings

---
 transformers/modeling_roberta.py    | 6 +++++-
 transformers/modeling_tf_roberta.py | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index 6b8d381579..9d16c87888 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -478,12 +478,16 @@ class RobertaForTokenClassification(BertPreTrainedModel):
 
         tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
         model = RobertaForTokenClassification.from_pretrained('roberta-base')
-        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
         labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
         outputs = model(input_ids, labels=labels)
         loss, scores = outputs[:2]
 
     """
+    config_class = RobertaConfig
+    pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "roberta"
+
     def __init__(self, config):
         super(RobertaForTokenClassification, self).__init__(config)
         self.num_labels = config.num_labels
diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py
index 13a0522211..a239bc642b 100644
--- a/transformers/modeling_tf_roberta.py
+++ b/transformers/modeling_tf_roberta.py
@@ -396,7 +396,7 @@ class TFRobertaForTokenClassification(TFRobertaPreTrainedModel):
 
         tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
         model = TFRobertaForTokenClassification.from_pretrained('roberta-base')
-        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :]  # Batch size 1
         outputs = model(input_ids)
         scores = outputs[0]
 

From 4e5f88b74fa914a5f45aec3260977acfc3513536 Mon Sep 17 00:00:00 2001
From: Matt Maybeno <mmaybeno@gmail.com>
Date: Wed, 23 Oct 2019 22:50:03 -0700
Subject: [PATCH 086/269] Add Roberta to run_ner.py

---
 examples/run_ner.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index 00eb039258..16fa89c3e7 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -35,15 +35,17 @@ from utils_ner import convert_examples_to_features, get_labels, read_examples_fr
 
 from transformers import AdamW, WarmupLinearSchedule
 from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer
+from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer
 
 logger = logging.getLogger(__name__)
 
 ALL_MODELS = sum(
-    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, )),
+    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig)),
     ())
 
 MODEL_CLASSES = {
     "bert": (BertConfig, BertForTokenClassification, BertTokenizer),
+    "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer)
 }
 
 

From ae1d03fc51bb22ed59517ee6f92c560417fdb049 Mon Sep 17 00:00:00 2001
From: Matt Maybeno <mmaybeno@gmail.com>
Date: Thu, 24 Oct 2019 10:43:57 -0700
Subject: [PATCH 087/269] Add roberta to doc

---
 examples/run_ner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index 16fa89c3e7..740b422429 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Fine-tuning the library models for named entity recognition on CoNLL-2003 (Bert). """
+""" Fine-tuning the library models for named entity recognition on CoNLL-2003 (Bert or Roberta). """
 
 from __future__ import absolute_import, division, print_function
 

From bab6ad01aad6f6a8cbf5b8634d890d8fce9f46d1 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 24 Oct 2019 21:41:45 +0000
Subject: [PATCH 088/269] run_tf_glue works with all tasks

---
 examples/run_tf_glue.py               | 43 ++++++++++++++++++++++-----
 transformers/data/processors/glue.py  |  4 +++
 transformers/data/processors/utils.py |  7 +++++
 3 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py
index 399fe9e616..73173b0cf1 100644
--- a/examples/run_tf_glue.py
+++ b/examples/run_tf_glue.py
@@ -1,29 +1,47 @@
 import os
 import tensorflow as tf
 import tensorflow_datasets
-from transformers import BertTokenizer, TFBertForSequenceClassification, glue_convert_examples_to_features, BertForSequenceClassification
+from transformers import BertTokenizer, TFBertForSequenceClassification, BertConfig, glue_convert_examples_to_features, BertForSequenceClassification, glue_processors
 
 # script parameters
 BATCH_SIZE = 32
 EVAL_BATCH_SIZE = BATCH_SIZE * 2
 USE_XLA = False
 USE_AMP = False
+EPOCHS = 3
+
+TASK = "mrpc"
+
+if TASK == "sst-2":
+    TFDS_TASK = "sst2"
+elif TASK == "sts-b":
+    TFDS_TASK = "stsb"
+else: 
+    TFDS_TASK = TASK
+
+num_labels = len(glue_processors[TASK]().get_labels())
+print(num_labels)
 
 tf.config.optimizer.set_jit(USE_XLA)
 tf.config.optimizer.set_experimental_options({"auto_mixed_precision": USE_AMP})
 
-# Load tokenizer and model from pretrained model/vocabulary
+# Load tokenizer and model from pretrained model/vocabulary. Specify the number of labels to classify (2+: classification, 1: regression)
+config = BertConfig.from_pretrained("bert-base-cased", num_labels=num_labels)
 tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
-model = TFBertForSequenceClassification.from_pretrained('bert-base-cased')
+model = TFBertForSequenceClassification.from_pretrained('bert-base-cased', config=config)
 
 # Load dataset via TensorFlow Datasets
-data, info = tensorflow_datasets.load('glue/mrpc', with_info=True)
+data, info = tensorflow_datasets.load(f'glue/{TFDS_TASK}', with_info=True)
 train_examples = info.splits['train'].num_examples
+
+# MNLI expects either validation_matched or validation_mismatched
 valid_examples = info.splits['validation'].num_examples
 
 # Prepare dataset for GLUE as a tf.data.Dataset instance
-train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, 128, 'mrpc')
-valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, 128, 'mrpc')
+train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, 128, TASK)
+
+# MNLI expects either validation_matched or validation_mismatched
+valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, 128, TASK)
 train_dataset = train_dataset.shuffle(128).batch(BATCH_SIZE).repeat(-1)
 valid_dataset = valid_dataset.batch(EVAL_BATCH_SIZE)
 
@@ -32,7 +50,13 @@ opt = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08)
 if USE_AMP:
     # loss scaling is currently required when using mixed precision
     opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic')
-loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+
+
+if num_labels == 1:
+    loss = tf.keras.losses.MeanSquaredError()
+else:
+    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+
 metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')
 model.compile(optimizer=opt, loss=loss, metrics=[metric])
 
@@ -40,7 +64,7 @@ model.compile(optimizer=opt, loss=loss, metrics=[metric])
 train_steps = train_examples//BATCH_SIZE
 valid_steps = valid_examples//EVAL_BATCH_SIZE
 
-history = model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps,
+history = model.fit(train_dataset, epochs=EPOCHS, steps_per_epoch=train_steps,
                     validation_data=valid_dataset, validation_steps=valid_steps)
 
 # Save TF2 model
@@ -57,6 +81,9 @@ sentence_2 = 'His findings were not compatible with this research.'
 inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')
 inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt')
 
+del inputs_1["special_tokens_mask"]
+del inputs_2["special_tokens_mask"]
+
 pred_1 = pytorch_model(**inputs_1)[0].argmax().item()
 pred_2 = pytorch_model(**inputs_2)[0].argmax().item()
 print('sentence_1 is', 'a paraphrase' if pred_1 else 'not a paraphrase', 'of sentence_0')
diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py
index 741569ea30..c81582fb72 100644
--- a/transformers/data/processors/glue.py
+++ b/transformers/data/processors/glue.py
@@ -76,10 +76,14 @@ def glue_convert_examples_to_features(examples, tokenizer,
 
     features = []
     for (ex_index, example) in enumerate(examples):
+        if ex_index == 10:
+            break
+        
         if ex_index % 10000 == 0:
             logger.info("Writing example %d" % (ex_index))
         if is_tf_dataset:
             example = processor.get_example_from_tensor_dict(example)
+            example = processor.tfds_map(example)
 
         inputs = tokenizer.encode_plus(
             example.text_a,
diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py
index 27138f9959..07bdf3150c 100644
--- a/transformers/data/processors/utils.py
+++ b/transformers/data/processors/utils.py
@@ -107,6 +107,13 @@ class DataProcessor(object):
         """Gets the list of labels for this data set."""
         raise NotImplementedError()
 
+    def tfds_map(self, example):
+        """Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. 
+        This method converts examples to the correct format."""
+        if len(self.get_labels()) > 1:
+            example.label = self.get_labels()[int(example.label)]
+        return example
+
     @classmethod
     def _read_tsv(cls, input_file, quotechar=None):
         """Reads a tab separated value file."""

From beaf66b1f30aa29e11a02ecb5a7edb6b7b99eb01 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 24 Oct 2019 21:43:28 +0000
Subject: [PATCH 089/269] Remove break

---
 transformers/data/processors/glue.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py
index c81582fb72..518251b050 100644
--- a/transformers/data/processors/glue.py
+++ b/transformers/data/processors/glue.py
@@ -76,9 +76,6 @@ def glue_convert_examples_to_features(examples, tokenizer,
 
     features = []
     for (ex_index, example) in enumerate(examples):
-        if ex_index == 10:
-            break
-        
         if ex_index % 10000 == 0:
             logger.info("Writing example %d" % (ex_index))
         if is_tf_dataset:

From f873a3edb2eba78d92e55ecb55902f7e9cb90777 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 15:21:46 +0200
Subject: [PATCH 090/269] the decoder attends to the output of the encoder
 stack (last layer)

---
 transformers/modeling_bert.py    | 13 ++++++-------
 transformers/modeling_seq2seq.py |  2 +-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index be8ec5ba21..aa022bac8a 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -288,8 +288,8 @@ class BertAttention(nn.Module):
         self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None, encoder_attention_mask=None):
-        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_state, encoder_attention_mask)
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
+        self_outputs = self.self(hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)
         attention_output = self.output(self_outputs[0], hidden_states)
         outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
         return outputs
@@ -334,13 +334,13 @@ class BertLayer(nn.Module):
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_state=None, encoder_attention_mask=None):
+    def forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
         self_attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
         attention_output = self_attention_outputs[0]
         outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
 
-        if self.is_decoder and encoder_hidden_state is not None:
-            cross_attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_state, encoder_attention_mask)
+        if self.is_decoder and encoder_hidden_states is not None:
+            cross_attention_outputs = self.crossattention(attention_output, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)
             attention_output = cross_attention_outputs[0]
             outputs = outputs + cross_attention_outputs[1:]  # add cross attentions if we output attention weights
 
@@ -364,8 +364,7 @@ class BertEncoder(nn.Module):
             if self.output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
-            encoder_hidden_state = encoder_hidden_states[i]
-            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_state, encoder_attention_mask)
+            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i], encoder_hidden_states, encoder_attention_mask)
             hidden_states = layer_outputs[0]
 
             if self.output_attentions:
diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index ca3b9dc87a..108fdaa853 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -165,7 +165,7 @@ class PreTrainedSeq2seq(nn.Module):
         encoder_hidden_states = kwargs_encoder.pop("encoder_hidden_states", None)
         if encoder_hidden_states is None:
             encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder)
-            encoder_hidden_states = encoder_outputs[0]
+            encoder_hidden_states = encoder_outputs[0][-1]  # output of the encoder *stack*
         else:
             encoder_outputs = ()
 

From dc580dd4c720c5daefe7411f604b6908da99681e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 16:56:36 +0200
Subject: [PATCH 091/269] add lm_labels for the LM cross-entropy

---
 transformers/modeling_bert.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index aa022bac8a..d10f32c1fa 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -819,7 +819,7 @@ class BertForMaskedLM(BertPreTrainedModel):
                                    self.bert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None):
+                masked_lm_labels=None, lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
@@ -840,7 +840,7 @@ class BertForMaskedLM(BertPreTrainedModel):
         #    of predictions for masked words.
         # 2. If encoder hidden states are provided we are in a causal situation where we
         #    try to predict the next word for each input in the encoder.
-        if masked_lm_labels is not None and encoder_hidden_states is not None:
+        if masked_lm_labels is not None and lm_labels is not None:
             raise AttributeError("Masked LM training with an encoder-decoder is not supported.")
 
         if masked_lm_labels is not None:
@@ -848,12 +848,12 @@ class BertForMaskedLM(BertPreTrainedModel):
             masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
             outputs = (masked_lm_loss,) + outputs
 
-        if encoder_hidden_states is not None:
+        if lm_labels is not None:
             # we are doing next-token prediction; shift prediction scores and input ids by one
             prediction_scores = prediction_scores[:, :-1, :]
-            input_ids = input_ids[:, 1:, :]
+            lm_labels = lm_labels[:, 1:, :]
             loss_fct = CrossEntropyLoss(ignore_index=-1)
-            seq2seq_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), input_ids.view(-1))
+            seq2seq_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), lm_labels.view(-1))
             outputs = (seq2seq_loss,) + outputs
 
         return outputs  # (mlm_or_seq2seq_loss), prediction_scores, (hidden_states), (attentions)

From b915ba9dfe51db8161db5bc599df3944646b2b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 17:44:20 +0200
Subject: [PATCH 092/269] pad sequence with 0, mask with -1

---
 examples/run_seq2seq_finetuning.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 38dcb2d005..1f21cff82c 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -58,7 +58,7 @@ class TextDataset(Dataset):
     [2] https://github.com/abisee/cnn-dailymail/
     """
 
-    def __init__(self, tokenizer, prefix='train', data_dir="", block_size=512):
+    def __init__(self, tokenizer, prefix="train", data_dir="", block_size=512):
         assert os.path.isdir(data_dir)
 
         # Load features that have already been computed if present
@@ -165,7 +165,12 @@ def _fit_to_block_size(sequence, block_size):
     if len(sequence) > block_size:
         return sequence[:block_size]
     else:
-        return sequence.extend([-1] * (block_size - len(sequence)))
+        return sequence.extend([0] * (block_size - len(sequence)))
+
+
+def mask_padding_tokens(sequence):
+    """ Replace the padding token with -1 values """
+    return [s if s != 0 else -1 for s in sequence]
 
 
 def load_and_cache_examples(args, tokenizer):
@@ -219,11 +224,8 @@ def train(args, train_dataset, model, tokenizer):
     logger.info("***** Running training *****")
     logger.info("  Num examples = %d", len(train_dataset))
     logger.info("  Num Epochs = %d", args.num_train_epochs)
-    logger.info(
-        "  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size
-    )
-    logger.info(
-        "  Total train batch size (w. parallel, distributed & accumulation) = %d",
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
         args.train_batch_size
         * args.gradient_accumulation_steps
         * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
@@ -242,7 +244,7 @@ def train(args, train_dataset, model, tokenizer):
             source = ([s for s, _ in batch]).to(args.device)
             target = ([t for _, t in batch]).to(args.device)
             model.train()
-            outputs = model(source, target)
+            outputs = model(source, target, decoder_lm_labels=mask_padding_tokens(target))
             loss = outputs[0]
             loss.backward()
 

From cb26b035c696f32b7f47df18a6d84b88b7b1745d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 17:52:32 +0200
Subject: [PATCH 093/269] remove potential UndefinedError

---
 transformers/modeling_xlm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py
index f1df6f668f..166b98de63 100644
--- a/transformers/modeling_xlm.py
+++ b/transformers/modeling_xlm.py
@@ -81,8 +81,8 @@ def get_masks(slen, lengths, causal, padding_mask=None):
         mask = alen < lengths[:, None]
 
     # attention mask is the same as mask, or triangular inferior attention (causal)
+    bs = lengths.size(0)
     if causal:
-        bs = lengths.size(0)
         attn_mask = alen[None, None, :].repeat(bs, slen, 1) <= alen[None, :, None]
     else:
         attn_mask = mask

From a67413ccc82ed6bfdf9ea2f6b17ee3869f2f87a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 18:08:09 +0200
Subject: [PATCH 094/269] extend works in-place

---
 examples/run_seq2seq_finetuning.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
index 1f21cff82c..61c4abfe6e 100644
--- a/examples/run_seq2seq_finetuning.py
+++ b/examples/run_seq2seq_finetuning.py
@@ -165,7 +165,8 @@ def _fit_to_block_size(sequence, block_size):
     if len(sequence) > block_size:
         return sequence[:block_size]
     else:
-        return sequence.extend([0] * (block_size - len(sequence)))
+        sequence.extend([0] * (block_size - len(sequence)))
+        return sequence
 
 
 def mask_padding_tokens(sequence):

From 932543f77ee69b776a2ea4c09f09745624d4c6a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 17 Oct 2019 21:07:55 +0200
Subject: [PATCH 095/269] fix test of truncation function

---
 examples/run_seq2seq_finetuning_test.py | 48 +++++++------------------
 1 file changed, 13 insertions(+), 35 deletions(-)

diff --git a/examples/run_seq2seq_finetuning_test.py b/examples/run_seq2seq_finetuning_test.py
index e59f016da4..77dc58666c 100644
--- a/examples/run_seq2seq_finetuning_test.py
+++ b/examples/run_seq2seq_finetuning_test.py
@@ -21,43 +21,21 @@ class DataLoaderTest(unittest.TestCase):
     def setUp(self):
         self.block_size = 10
 
-    def test_truncate_source_and_target_too_small(self):
-        """ When the sum of the lengths of the source and target sequences is
-        smaller than the block size (minus the number of special tokens), skip the example. """
-        src_seq = [1, 2, 3, 4]
-        tgt_seq = [5, 6]
-        self.assertEqual(_fit_to_block_size(src_seq, tgt_seq, self.block_size), None)
+    def test_truncate_sequence_too_small(self):
+        """ Pad the sequence with 0 if the sequence is smaller than the block size."""
+        sequence = [1, 2, 3, 4]
+        expected_output = [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
+        self.assertEqual(_fit_to_block_size(sequence, self.block_size), expected_output)
 
-    def test_truncate_source_and_target_fit_exactly(self):
-        """ When the sum of the lengths of the source and target sequences is
-        equal to the block size (minus the number of special tokens), return the
-        sequences unchanged. """
-        src_seq = [1, 2, 3, 4]
-        tgt_seq = [5, 6, 7]
-        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(src_seq, fitted_src)
-        self.assertListEqual(tgt_seq, fitted_tgt)
+    def test_truncate_sequence_fit_exactly(self):
+        sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        self.assertEqual(_fit_to_block_size(sequence, self.block_size), expected_output)
 
-    def test_truncate_source_too_big_target_ok(self):
-        src_seq = [1, 2, 3, 4, 5, 6]
-        tgt_seq = [1, 2]
-        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(fitted_src, [1, 2, 3, 4, 5])
-        self.assertListEqual(fitted_tgt, fitted_tgt)
-
-    def test_truncate_target_too_big_source_ok(self):
-        src_seq = [1, 2, 3, 4]
-        tgt_seq = [1, 2, 3, 4]
-        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(fitted_src, src_seq)
-        self.assertListEqual(fitted_tgt, [1, 2, 3])
-
-    def test_truncate_source_and_target_too_big(self):
-        src_seq = [1, 2, 3, 4, 5, 6, 7]
-        tgt_seq = [1, 2, 3, 4, 5, 6, 7]
-        fitted_src, fitted_tgt = _fit_to_block_size(src_seq, tgt_seq, self.block_size)
-        self.assertListEqual(fitted_src, [1, 2, 3, 4, 5])
-        self.assertListEqual(fitted_tgt, [1, 2])
+    def test_truncate_sequence_too_big(self):
+        sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+        expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        self.assertEqual(_fit_to_block_size(sequence, self.block_size), expected_output)
 
     def test_process_story_no_highlights(self):
         """ Processing a story with no highlights should raise an exception.

From 4c3ac4a7d83cdf37b796d783bb66a89bbd09ef9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Fri, 18 Oct 2019 12:29:30 +0200
Subject: [PATCH 096/269] here's one big commit

---
 examples/README.md                            |   5 +-
 examples/run_seq2seq_finetuning.py            | 361 ----------
 examples/run_summarization_finetuning.py      | 620 ++++++++++++++++++
 ...y => run_summarization_finetuning_test.py} |  28 +-
 transformers/__init__.py                      |   2 +-
 transformers/modeling_beam_search.py          | 240 +++++++
 transformers/modeling_bert.py                 |  20 +-
 transformers/modeling_seq2seq.py              |  83 ++-
 8 files changed, 951 insertions(+), 408 deletions(-)
 delete mode 100644 examples/run_seq2seq_finetuning.py
 create mode 100644 examples/run_summarization_finetuning.py
 rename examples/{run_seq2seq_finetuning_test.py => run_summarization_finetuning_test.py} (79%)
 create mode 100644 transformers/modeling_beam_search.py

diff --git a/examples/README.md b/examples/README.md
index e0fe1fc704..bec6d57171 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -393,7 +393,8 @@ This fine-tuned model is available as a checkpoint under the reference
 
 ## Seq2seq model fine-tuning
 
-Based on the script [`run_seq2seq_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_seq2seq_finetuning.py).
+Based on the script
+[`run_summarization_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_summarization_finetuning.py).
 
 Before running this script you should download **both** CNN and Daily Mail
 datasets from [Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/)  (the
@@ -412,7 +413,7 @@ archive.
 ```bash
 export DATA_PATH=/path/to/dataset/
 
-python run_seq2seq_finetuning.py \
+python run_summarization_finetuning.py \
     --output_dir=output \
     --model_type=bert2bert \
     --model_name_or_path=bert2bert \
diff --git a/examples/run_seq2seq_finetuning.py b/examples/run_seq2seq_finetuning.py
deleted file mode 100644
index 61c4abfe6e..0000000000
--- a/examples/run_seq2seq_finetuning.py
+++ /dev/null
@@ -1,361 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Microsoft Reseach team and The HuggingFace Inc. team.
-# Copyright (c) 2018 Microsoft and The HuggingFace Inc.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Finetuning seq2seq models for sequence generation."""
-
-import argparse
-from collections import deque
-import logging
-import pickle
-import random
-import os
-
-import numpy as np
-from tqdm import tqdm, trange
-import torch
-from torch.utils.data import Dataset, RandomSampler
-
-from transformers import AutoTokenizer, Model2Model
-
-logger = logging.getLogger(__name__)
-
-
-def set_seed(args):
-    random.seed(args.seed)
-    np.random.seed(args.seed)
-    torch.manual_seed(args.seed)
-
-
-# ------------
-# Load dataset
-# ------------
-
-
-class TextDataset(Dataset):
-    """ Abstracts the dataset used to train seq2seq models.
-
-    CNN/Daily News:
-
-    The CNN/Daily News raw datasets are downloaded from [1]. The stories are
-    stored in different files; the summary appears at the end of the story as
-    sentences that are prefixed by the special `@highlight` line. To process
-    the data, untar both datasets in the same folder, and pass the path to this
-    folder as the "data_dir argument. The formatting code was inspired by [2].
-
-    [1] https://cs.nyu.edu/~kcho/
-    [2] https://github.com/abisee/cnn-dailymail/
-    """
-
-    def __init__(self, tokenizer, prefix="train", data_dir="", block_size=512):
-        assert os.path.isdir(data_dir)
-
-        # Load features that have already been computed if present
-        cached_features_file = os.path.join(
-            data_dir, "cached_lm_{}_{}".format(block_size, prefix)
-        )
-        if os.path.exists(cached_features_file):
-            logger.info("Loading features from cached file %s", cached_features_file)
-            with open(cached_features_file, "rb") as source:
-                self.examples = pickle.load(source)
-                return
-
-        logger.info("Creating features from dataset at %s", data_dir)
-        self.examples = []
-        datasets = ["cnn", "dailymail"]
-        for dataset in datasets:
-            path_to_stories = os.path.join(data_dir, dataset, "stories")
-            assert os.path.isdir(path_to_stories)
-
-            story_filenames_list = os.listdir(path_to_stories)
-            for story_filename in story_filenames_list:
-                path_to_story = os.path.join(path_to_stories, story_filename)
-                if not os.path.isfile(path_to_story):
-                    continue
-
-                with open(path_to_story, encoding="utf-8") as source:
-                    try:
-                        raw_story = source.read()
-                        story, summary = process_story(raw_story)
-                    except IndexError:  # skip ill-formed stories
-                        continue
-
-                story = tokenizer.encode(story)
-                story_seq = _fit_to_block_size(story, block_size)
-
-                summary = tokenizer.encode(summary)
-                summary_seq = _fit_to_block_size(summary, block_size)
-
-                self.examples.append((story_seq, summary_seq))
-
-        logger.info("Saving features into cache file %s", cached_features_file)
-        with open(cached_features_file, "wb") as sink:
-            pickle.dump(self.examples, sink, protocol=pickle.HIGHEST_PROTOCOL)
-
-    def __len__(self):
-        return len(self.examples)
-
-    def __getitem__(self, items):
-        return torch.tensor(self.examples[items])
-
-
-def process_story(raw_story):
-    """ Extract the story and summary from a story file.
-
-    Attributes:
-        raw_story (str): content of the story file as an utf-8 encoded string.
-
-    Raises:
-        IndexError: If the stoy is empty or contains no highlights.
-    """
-    file_lines = list(
-        filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")])
-    )
-
-    # for some unknown reason some lines miss a period, add it
-    file_lines = [_add_missing_period(line) for line in file_lines]
-
-    # gather article lines
-    story_lines = []
-    lines = deque(file_lines)
-    while True:
-        try:
-            element = lines.popleft()
-            if element.startswith("@highlight"):
-                break
-            story_lines.append(element)
-        except IndexError as ie:  # if "@highlight" absent from file
-            raise ie
-
-    # gather summary lines
-    highlights_lines = list(filter(lambda t: not t.startswith("@highlight"), lines))
-
-    # join the lines
-    story = " ".join(story_lines)
-    summary = " ".join(highlights_lines)
-
-    return story, summary
-
-
-def _add_missing_period(line):
-    END_TOKENS = [".", "!", "?", "...", "'", "`", '"', u"\u2019", u"\u2019", ")"]
-    if line.startswith("@highlight"):
-        return line
-    if line[-1] in END_TOKENS:
-        return line
-    return line + "."
-
-
-def _fit_to_block_size(sequence, block_size):
-    """ Adapt the source and target sequences' lengths to the block size.
-    If the sequence is shorter than the block size we pad it with -1 ids
-    which correspond to padding tokens.
-    """
-    if len(sequence) > block_size:
-        return sequence[:block_size]
-    else:
-        sequence.extend([0] * (block_size - len(sequence)))
-        return sequence
-
-
-def mask_padding_tokens(sequence):
-    """ Replace the padding token with -1 values """
-    return [s if s != 0 else -1 for s in sequence]
-
-
-def load_and_cache_examples(args, tokenizer):
-    dataset = TextDataset(tokenizer, data_dir=args.data_dir)
-    return dataset
-
-
-# ------------
-# Train
-# ------------
-
-
-def train(args, train_dataset, model, tokenizer):
-    """ Fine-tune the pretrained model on the corpus. """
-
-    # Prepare the data loading
-    args.train_bach_size = 1
-    train_sampler = RandomSampler(train_dataset)
-    train_dataloader = DataLoader(
-        train_dataset, sampler=train_sampler, batch_size=args.train_bach_size
-    )
-
-    # Prepare the optimizer and schedule (linear warmup and decay)
-    no_decay = ["bias", "LayerNorm.weight"]
-    optimizer_grouped_parameters = [
-        {
-            "params": [
-                p
-                for n, p in model.named_parameters()
-                if not any(nd in n for nd in no_decay)
-            ],
-            "weight_decay": args.weight_decay,
-        },
-        {
-            "params": [
-                p
-                for n, p in model.named_parameters()
-                if any(nd in n for nd in no_decay)
-            ],
-            "weight_decay": 0.0,
-        },
-    ]
-    optimizer = AdamW(
-        optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon
-    )
-    scheduler = WarmupLinearSchedule(
-        optimizer, warmup_steps=args.warmup_steps, t_total=t_total
-    )
-
-    # Train
-    logger.info("***** Running training *****")
-    logger.info("  Num examples = %d", len(train_dataset))
-    logger.info("  Num Epochs = %d", args.num_train_epochs)
-    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
-    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
-        args.train_batch_size
-        * args.gradient_accumulation_steps
-        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
-    )
-    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
-    logger.info("  Total optimization steps = %d", t_total)
-
-    global_step = 0
-    tr_loss, logging_loss = 0.0, 0.0
-    model.zero_grad()
-    train_iterator = trange(args.num_train_epochs, desc="Epoch", disable=True)
-    set_seed(args)
-    for _ in train_iterator:
-        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=True)
-        for step, batch in enumerate(epoch_iterator):
-            source = ([s for s, _ in batch]).to(args.device)
-            target = ([t for _, t in batch]).to(args.device)
-            model.train()
-            outputs = model(source, target, decoder_lm_labels=mask_padding_tokens(target))
-            loss = outputs[0]
-            loss.backward()
-
-            tr_loss += loss.item()
-            if (step + 1) % args.gradient_accumulation_steps == 0:
-                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
-                optimizer.step()
-                scheduler.step()
-                model.zero_grad()
-                global_step += 1
-
-            if args.max_steps > 0 and global_step > args.max_steps:
-                epoch_iterator.close()
-                break
-
-        if args.max_steps > 0 and global_step > args.max_steps:
-            train_iterator.close()
-            break
-
-    return global_step, tr_loss / global_step
-
-
-def main():
-    parser = argparse.ArgumentParser()
-
-    # Required parameters
-    parser.add_argument(
-        "--data_dir",
-        default=None,
-        type=str,
-        required=True,
-        help="The input training data file (a text file).",
-    )
-    parser.add_argument(
-        "--output_dir",
-        default=None,
-        type=str,
-        required=True,
-        help="The output directory where the model predictions and checkpoints will be written.",
-    )
-
-    # Optional parameters
-    parser.add_argument(
-        "--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer."
-    )
-    parser.add_argument(
-        "--model_name_or_path",
-        default="bert-base-cased",
-        type=str,
-        help="The model checkpoint to initialize the encoder and decoder's weights with.",
-    )
-    parser.add_argument(
-        "--model_type",
-        default="bert",
-        type=str,
-        help="The decoder architecture to be fine-tuned.",
-    )
-    parser.add_argument(
-        "--learning_rate",
-        default=5e-5,
-        type=float,
-        help="The initial learning rate for Adam.",
-    )
-    parser.add_argument(
-        "--max_grad_norm", default=1.0, type=float, help="Max gradient norm."
-    )
-    parser.add_argument(
-        "--max_steps",
-        default=-1,
-        type=int,
-        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
-    )
-    parser.add_argument(
-        "--num_train_epochs",
-        default=1,
-        type=int,
-        help="Total number of training epochs to perform.",
-    )
-    parser.add_argument("--seed", default=42, type=int)
-    parser.add_argument(
-        "--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps."
-    )
-    parser.add_argument(
-        "--weight_decay", default=0.0, type=float, help="Weight deay if we apply some."
-    )
-    args = parser.parse_args()
-
-    if args.model_type != "bert":
-        raise ValueError(
-            "Only the BERT architecture is currently supported for seq2seq."
-        )
-
-    # Set up training device
-    # device = torch.device("cpu")
-
-    # Set seed
-    set_seed(args)
-
-    # Load pretrained model and tokenizer
-    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
-    model = Model2Model.from_pretrained(args.model_name_or_path)
-    # model.to(device)
-
-    logger.info("Training/evaluation parameters %s", args)
-
-    # Training
-    train_dataset = load_and_cache_examples(args, tokenizer)
-    global_step, tr_loss = train(args, train_dataset, model, tokenizer)
-    # logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py
new file mode 100644
index 0000000000..64bee82c5b
--- /dev/null
+++ b/examples/run_summarization_finetuning.py
@@ -0,0 +1,620 @@
+# coding=utf-8
+# Copyright 2019 The HuggingFace Inc. team.
+# Copyright (c) 2019 The HuggingFace Inc.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning seq2seq models for sequence generation."""
+
+import argparse
+from collections import deque
+import logging
+import os
+import pickle
+import random
+import sys
+
+import numpy as np
+from tqdm import tqdm, trange
+import torch
+from torch.optim import Adam
+from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
+
+from transformers import AutoTokenizer, PreTrainedSeq2seq, Model2Model
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+
+
+# ------------
+# Load dataset
+# ------------
+
+
+class TextDataset(Dataset):
+    """ Abstracts the dataset used to train seq2seq models.
+
+    CNN/Daily News:
+
+    The CNN/Daily News raw datasets are downloaded from [1]. The stories are
+    stored in different files; the summary appears at the end of the story as
+    sentences that are prefixed by the special `@highlight` line. To process
+    the data, untar both datasets in the same folder, and pass the path to this
+    folder as the "data_dir argument. The formatting code was inspired by [2].
+
+    [1] https://cs.nyu.edu/~kcho/
+    [2] https://github.com/abisee/cnn-dailymail/
+    """
+
+    def __init__(self, tokenizer, prefix="train", data_dir="", block_size=512):
+        assert os.path.isdir(data_dir)
+
+        # Load the features that have already been computed, if any
+        cached_features_file = os.path.join(
+            data_dir, "cached_lm_{}_{}".format(block_size, prefix)
+        )
+        if os.path.exists(cached_features_file):
+            logger.info("Loading features from cached file %s", cached_features_file)
+            with open(cached_features_file, "rb") as source:
+                self.examples = pickle.load(source)
+                return
+
+        logger.info("Creating features from dataset at %s", data_dir)
+        datasets = ["cnn", "dailymail"]
+
+        self.examples = {"source": [], "target": []}
+        for dataset in datasets:
+            path_to_stories = os.path.join(data_dir, dataset, "stories")
+            story_filenames_list = os.listdir(path_to_stories)
+            for story_filename in story_filenames_list:
+                path_to_story = os.path.join(path_to_stories, story_filename)
+                if not os.path.isfile(path_to_story):
+                    continue
+
+                with open(path_to_story, encoding="utf-8") as source:
+                    raw_story = source.read()
+                    story_lines, summary_lines = process_story(raw_story)
+                    if len(summary_lines) == 0 or len(story_lines) == 0:
+                        continue
+
+                story_token_ids, summary_token_ids = _encode_for_summarization(
+                    story_lines, summary_lines, tokenizer
+                )
+                story_seq = _fit_to_block_size(story_token_ids, block_size)
+                self.examples["source"].append(story_seq)
+
+                summary_seq = _fit_to_block_size(summary_token_ids, block_size)
+                self.examples["summary"].append(summary_seq)
+
+        logger.info("Saving features into cache file %s", cached_features_file)
+        with open(cached_features_file, "wb") as sink:
+            pickle.dump(self.examples, sink, protocol=pickle.HIGHEST_PROTOCOL)
+
+    def __len__(self):
+        return len(self.examples)
+
+    def __getitem__(self, items):
+        return (
+            torch.tensor(self.examples["source"][items]),
+            torch.tensor(self.examples["target"][items]),
+        )
+
+
+def process_story(raw_story):
+    """ Extract the story and summary from a story file.
+
+    Attributes:
+        raw_story (str): content of the story file as an utf-8 encoded string.
+
+    Raises:
+        IndexError: If the stoy is empty or contains no highlights.
+    """
+    nonempty_lines = list(
+        filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")])
+    )
+
+    # for some unknown reason some lines miss a period, add it
+    nonempty_lines = [_add_missing_period(line) for line in nonempty_lines]
+
+    # gather article lines
+    story_lines = []
+    lines = deque(nonempty_lines)
+    while True:
+        try:
+            element = lines.popleft()
+            if element.startswith("@highlight"):
+                break
+            story_lines.append(element)
+        except IndexError:
+            # if "@highlight" is absent from the file we pop
+            # all elements until there is None.
+            return story_lines, []
+
+    # gather summary lines
+    summary_lines = list(filter(lambda t: not t.startswith("@highlight"), lines))
+
+    return story_lines, summary_lines
+
+
+def _encode_for_summarization(story_lines, summary_lines, tokenizer):
+    """ Encode the story and summary lines, and join them
+    as specified in [1] by using `[SEP] [CLS]` tokens to separate
+    sentences.
+    """
+    story_lines_token_ids = [
+        tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line))
+        for line in story_lines
+    ]
+    summary_lines_token_ids = [
+        tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line))
+        for line in summary_lines
+    ]
+
+    story_token_ids = [
+        token for sentence in story_lines_token_ids for token in sentence
+    ]
+    summary_token_ids = [
+        token for sentence in summary_lines_token_ids for token in sentence
+    ]
+
+    return story_token_ids, summary_token_ids
+
+
+def _add_missing_period(line):
+    END_TOKENS = [".", "!", "?", "...", "'", "`", '"', u"\u2019", u"\u2019", ")"]
+    if line.startswith("@highlight"):
+        return line
+    if line[-1] in END_TOKENS:
+        return line
+    return line + "."
+
+
+def _fit_to_block_size(sequence, block_size):
+    """ Adapt the source and target sequences' lengths to the block size.
+    If the sequence is shorter than the block size we pad it with -1 ids
+    which correspond to padding tokens.
+    """
+    if len(sequence) > block_size:
+        return sequence[:block_size]
+    else:
+        sequence.extend([0] * (block_size - len(sequence)))
+        return sequence
+
+
+def mask_padding_tokens(sequence):
+    """ Padding token, encoded as 0, are represented by the value -1 in the
+    masks """
+    padded = sequence.clone()
+    padded[padded == 0] = -1
+    return padded
+
+
+def load_and_cache_examples(args, tokenizer):
+    dataset = TextDataset(tokenizer, data_dir=args.data_dir)
+    return dataset
+
+
+def compute_token_type_ids(batch, separator_token_id):
+    """ Segment embeddings as described in [1]
+
+    The values {0,1} were found in the repository [2].
+
+    Attributes:
+        batch: torch.Tensor, size [batch_size, block_size]
+            Batch of input.
+        separator_token_id: int
+            The value of the token that separates the segments.
+
+    [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders."
+        arXiv preprint arXiv:1908.08345 (2019).
+    [2] https://github.com/nlpyang/PreSumm (/src/prepro/data_builder.py, commit fac1217)
+    """
+    batch_embeddings = []
+    sentence_num = 0
+    for sequence in batch:
+        embeddings = []
+        for s in sequence:
+            if s == separator_token_id:
+                sentence_num += 1
+            embeddings.append(sentence_num % 2)
+        batch_embeddings.append(embeddings)
+    return torch.tensor(batch_embeddings)
+
+
+# ----------
+# Optimizers
+# ----------
+
+
+class BertSumOptimizer(object):
+    """ Specific optimizer for BertSum.
+
+    As described in [1], the authors fine-tune BertSum for abstractive
+    summarization using two Adam Optimizers with different warm-up steps and
+    learning rate. They also use a custom learning rate scheduler.
+
+    [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders."
+        arXiv preprint arXiv:1908.08345 (2019).
+    """
+
+    def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-9):
+        self.encoder = model.encoder
+        self.decoder = model.decoder
+        self.lr = lr
+        self.warmup_steps = warmup_steps
+
+        self.optimizers = {
+            "encoder": Adam(
+                model.encoder.parameters(),
+                lr=lr["encoder"],
+                betas=(beta_1, beta_2),
+                eps=eps,
+            ),
+            "decoder": Adam(
+                model.decoder.parameters(),
+                lr=lr["decoder"],
+                betas=(beta_1, beta_2),
+                eps=eps,
+            ),
+        }
+
+        self._step = 0
+
+    def _update_rate(self, stack):
+        return self.lr[stack] * min(
+            self._step ** (-0.5), self._step * self.warmup_steps[stack] ** (-0.5)
+        )
+
+    def zero_grad(self):
+        self.optimizer_decoder.zero_grad()
+        self.optimizer_encoder.zero_grad()
+
+    def step(self):
+        self._step += 1
+        for stack, optimizer in self.optimizers.items():
+            new_rate = self._update_rate(stack)
+            for param_group in optimizer.param_groups:
+                param_group["lr"] = new_rate
+            optimizer.step()
+
+
+# ------------
+# Train
+# ------------
+
+
+def train(args, model, tokenizer):
+    """ Fine-tune the pretrained model on the corpus. """
+    set_seed(args)
+
+    # Load the data
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_dataset = load_and_cache_examples(args, tokenizer)
+    train_sampler = RandomSampler(train_dataset)
+    train_dataloader = DataLoader(
+        train_dataset, sampler=train_sampler, batch_size=args.train_batch_size
+    )
+
+    # Training schedule
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = t_total // (
+            len(train_dataloader) // args.gradient_accumulation_steps + 1
+        )
+    else:
+        t_total = (
+            len(train_dataloader)
+            // args.gradient_accumulation_steps
+            * args.num_train_epochs
+        )
+
+    # Prepare the optimizer
+    lr = {"encoder": 0.002, "decoder": 0.2}
+    warmup_steps = {"encoder": 20000, "decoder": 10000}
+    optimizer = BertSumOptimizer(model, lr, warmup_steps)
+
+    # Train
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info(
+        "  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size
+    )
+    logger.info(
+        "  Total train batch size (w. parallel, distributed & accumulation) = %d",
+        args.train_batch_size * args.gradient_accumulation_steps
+        # * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
+    )
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    model.zero_grad()
+    train_iterator = trange(args.num_train_epochs, desc="Epoch", disable=True)
+
+    global_step = 0
+    tr_loss = 0.0
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=True)
+        for step, batch in enumerate(epoch_iterator):
+            source, target = batch
+            token_type_ids = compute_token_type_ids(source, tokenizer.cls_token_id)
+            labels_src = mask_padding_tokens(source)
+            labels_tgt = mask_padding_tokens(target)
+
+            source = source.to(args.device)
+            target = target.to(args.device)
+            token_type_ids = token_type_ids.to(args.device)
+            labels_src = labels_src.to(args.device)
+            labels_tgt = labels_tgt.to(args.device)
+
+            model.train()
+            outputs = model(
+                source,
+                target,
+                token_type_ids=token_type_ids,
+                decoder_encoder_attention_mask=labels_src,
+                decoder_attention_mask=labels_tgt,
+                decoder_lm_labels=labels_tgt,
+                decoder_initialize_randomly=True,
+            )
+
+            loss = outputs[0]
+            print(loss)
+            if args.gradient_accumulation_steps > 1:
+                loss /= args.gradient_accumulation_steps
+
+            loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+                optimizer.step()
+                model.zero_grad()
+                global_step += 1
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    return global_step, tr_loss / global_step
+
+
+# ------------
+# Train
+# ------------
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    set_seed(args)
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+    eval_dataset = load_and_cache_examples(args, tokenizer, evaluate=True)
+    eval_sampler = SequentialSampler(eval_dataset)
+    eval_dataloader = DataLoader(
+        eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size
+    )
+
+    logger.info("***** Running evaluation {} *****".format(prefix))
+    logger.info("  Num examples = %d", len(eval_dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    eval_loss = 0.0
+    nb_eval_steps = 0
+    model.eval()
+
+    for batch in tqdm(eval_dataloader, desc="Evaluating"):
+        source, target = batch
+        labels_src = mask_padding_tokens(source)
+        labels_tgt = mask_padding_tokens(target)
+        source.to(args.device)
+        target.to(args.device)
+        labels_src.to(args.device)
+        labels_tgt.to(args.device)
+
+        with torch.no_grad():
+            outputs = model(
+                source,
+                target,
+                decoder_encoder_attention_mask=labels_src,
+                decoder_attention_mask=labels_tgt,
+                decoder_lm_labels=labels_tgt,
+            )
+            lm_loss = outputs[0]
+            eval_loss += lm_loss.mean().item()
+        nb_eval_steps += 1
+
+    eval_loss = eval_loss / nb_eval_steps
+    perplexity = torch.exp(torch.tensor(eval_loss))
+
+    result = {"perplexity": perplexity}
+
+    # Save the evaluation's results
+    output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
+    if not os.path.exists(args.output_dir):
+        os.makedirs(args.output_dir)
+
+    with open(output_eval_file, "w") as writer:
+        logger.info("***** Eval results {} *****".format(prefix))
+        for key in sorted(result.keys()):
+            logger.info("  %s = %s", key, str(result[key]))
+            writer.write("%s = %s\n" % (key, str(result[key])))
+
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    # Required parameters
+    parser.add_argument(
+        "--data_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The input training data file (a text file).",
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
+
+    # Optional parameters
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument(
+        "--do_evaluate",
+        type=bool,
+        default=False,
+        help="Run model evaluation on out-of-sample data.",
+    )
+    parser.add_argument("--do_train", type=bool, default=False, help="Run training.")
+    parser.add_argument(
+        "--do_overwrite_output_dir",
+        type=bool,
+        default=False,
+        help="Whether to overwrite the output dir.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        default="bert-base-cased",
+        type=str,
+        help="The model checkpoint to initialize the encoder and decoder's weights with.",
+    )
+    parser.add_argument(
+        "--model_type",
+        default="bert",
+        type=str,
+        help="The decoder architecture to be fine-tuned.",
+    )
+    parser.add_argument(
+        "--max_grad_norm", default=1.0, type=float, help="Max gradient norm."
+    )
+    parser.add_argument(
+        "--max_steps",
+        default=-1,
+        type=int,
+        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
+    )
+    parser.add_argument(
+        "--to_cpu", default=False, type=bool, help="Whether to force training on CPU."
+    )
+    parser.add_argument(
+        "--num_train_epochs",
+        default=1,
+        type=int,
+        help="Total number of training epochs to perform.",
+    )
+    parser.add_argument(
+        "--per_gpu_train_batch_size",
+        default=4,
+        type=int,
+        help="Batch size per GPU/CPU for training.",
+    )
+    parser.add_argument("--seed", default=42, type=int)
+    args = parser.parse_args()
+
+    if (
+        os.path.exists(args.output_dir)
+        and os.listdir(args.output_dir)
+        and args.do_train
+        and not args.do_overwrite_output_dir
+    ):
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --do_overwrite_output_dir to overwrite.".format(
+                args.output_dir
+            )
+        )
+
+    # Set up training device
+    if args.to_cpu or not torch.cuda.is_available():
+        args.device = torch.device("cpu")
+        args.n_gpu = 0
+    else:
+        args.device = torch.device("cuda")
+        args.n_gpu = torch.cuda.device_count()
+
+    # Load pretrained model and tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
+    model = Model2Model.from_pretrained(args.model_name_or_path)
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        0,
+        args.device,
+        args.n_gpu,
+        False,
+        False,
+    )
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    # Train the model
+    model.to(args.device)
+    if args.do_train:
+        global_step, tr_loss = train(args, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+        if not os.path.exists(args.output_dir):
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = (
+            model.module if hasattr(model, "module") else model
+        )  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+        torch.save(args, os.path.join(args.output_dir, "training_arguments.bin"))
+
+    # Evaluate the model
+    results = {}
+    if args.do_evaluate:
+        checkpoints = []
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            encoder_checkpoint = os.path.join(checkpoint, "encoder")
+            decoder_checkpoint = os.path.join(checkpoint, "decoder")
+            model = PreTrainedSeq2seq.from_pretrained(
+                encoder_checkpoint, decoder_checkpoint
+            )
+            model.to(args.device)
+            results = "placeholder"
+
+    return results
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/run_seq2seq_finetuning_test.py b/examples/run_summarization_finetuning_test.py
similarity index 79%
rename from examples/run_seq2seq_finetuning_test.py
rename to examples/run_summarization_finetuning_test.py
index 77dc58666c..fd997ee0c2 100644
--- a/examples/run_seq2seq_finetuning_test.py
+++ b/examples/run_summarization_finetuning_test.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 import unittest
 
-from run_seq2seq_finetuning import _fit_to_block_size, process_story
+from run_summarization_finetuning import _fit_to_block_size, process_story
 
 
 class DataLoaderTest(unittest.TestCase):
@@ -43,15 +43,16 @@ class DataLoaderTest(unittest.TestCase):
         raw_story = """It was the year of Our Lord one thousand seven hundred and
         seventy-five.\n\nSpiritual revelations were conceded to England at that
         favoured period, as at this."""
-        with self.assertRaises(IndexError):
-            process_story(raw_story)
+        _, summary = process_story(raw_story)
+        self.assertEqual(summary, [])
 
     def test_process_empty_story(self):
         """ An empty story should also raise and exception.
         """
         raw_story = ""
-        with self.assertRaises(IndexError):
-            process_story(raw_story)
+        story, summary = process_story(raw_story)
+        self.assertEqual(story, [])
+        self.assertEqual(summary, [])
 
     def test_story_with_missing_period(self):
         raw_story = (
@@ -59,17 +60,16 @@ class DataLoaderTest(unittest.TestCase):
             "seventy-five\n\nSpiritual revelations were conceded to England "
             "at that favoured period, as at this.\n@highlight\n\nIt was the best of times"
         )
-        story, summary = process_story(raw_story)
+        story_lines, summary_lines = process_story(raw_story)
 
-        expected_story = (
-            "It was the year of Our Lord one thousand seven hundred and "
-            "seventy-five. Spiritual revelations were conceded to England at that "
-            "favoured period, as at this."
-        )
-        self.assertEqual(expected_story, story)
+        expected_story_lines = [
+            "It was the year of Our Lord one thousand seven hundred and seventy-five.",
+            "Spiritual revelations were conceded to England at that favoured period, as at this.",
+        ]
+        self.assertEqual(expected_story_lines, story_lines)
 
-        expected_summary = "It was the best of times."
-        self.assertEqual(expected_summary, summary)
+        expected_summary_lines = ["It was the best of times."]
+        self.assertEqual(expected_summary_lines, summary_lines)
 
 
 if __name__ == "__main__":
diff --git a/transformers/__init__.py b/transformers/__init__.py
index ee8e812a23..2206a0302e 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -87,7 +87,7 @@ if is_torch_available():
     from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel,
                                 DistilBertForSequenceClassification, DistilBertForQuestionAnswering,
                                 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
-    from .modeling_seq2seq import Model2Model
+    from .modeling_seq2seq import PreTrainedSeq2seq, Model2Model
 
     # Optimization
     from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule,
diff --git a/transformers/modeling_beam_search.py b/transformers/modeling_beam_search.py
new file mode 100644
index 0000000000..3a27625f90
--- /dev/null
+++ b/transformers/modeling_beam_search.py
@@ -0,0 +1,240 @@
+# coding=utf-8
+# Copyright (c) 2019 Yang Liu
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+A general wrapper around models with LM heads to generate sequences
+using beam search.
+"""
+import torch
+from torch import nn
+
+
+class ModelWithBeamSearch(nn.Module):
+    def __init__(
+        self,
+        model,
+        beam_size,
+        start_token_id,
+        end_token_id,
+        pad_token_id,
+        min_length,
+        max_length,
+        alpha,
+        block_trigram=True,
+    ):
+        """
+        Attributes:
+            mask_word_id: token id that corresponds to the mask
+        """
+        super(ModelWithBeamSearch, self).__init__()
+        self.model = model
+        self.beam_size = beam_size
+        self.start_token_id = start_token_id
+        self.end_token_id = end_token_id
+        self.pad_token_id = pad_token_id
+        self.min_length = min_length
+        self.max_length = max_length
+        self.alpha = alpha
+        self.block_trigram = block_trigram
+
+    def forward(self, input_ids, **kwargs):
+        # Separate the encoder- and decoder- specific kwargs. A kwarg is
+        # decoder-specific it the key starts with `decoder_`
+        kwargs_encoder = {
+            argument: value
+            for argument, value in kwargs.items()
+            if not argument.startswith("decoder_")
+        }
+        kwargs_decoder = {
+            argument[len("decoder_"):]: value
+            for argument, value in kwargs.items()
+            if argument.startswith("decoder_")
+        }
+
+        batch_size, _ = input_ids.size(0)
+
+        # Variables that keep track of the status of the search
+        hypotheses = [[] for _ in range(batch_size)]
+        batch_offset = torch.arange(batch_size, dtype=torch.long)
+        beam_offset = torch.arange(
+            0,
+            batch_size * self.beam_size,
+            step=self.beam_size,
+            dtype=torch.long,
+        )
+        growing_beam = torch.full(
+            (batch_size * self.beam_size, 1),
+            self.start_token_id,
+            dtype=torch.long,
+        )
+        topk_log_probabilities = torch.tensor(
+            [0.0] + [float("-inf")] * (self.beam_size - 1),
+            dtype=torch.float,
+        ).repeat(batch_size)
+
+        # Forward pass on the encoder
+        encoder_outputs = self.encoder(input_ids, kwargs_encoder)
+        kwargs_decoder["encoder_hidden_states"] = tile(
+            encoder_outputs, self.beam_size, dim=0
+        )
+
+        results = {}
+        results["predictions"] = [[] for _ in batch_size]
+        results["scores"] = [[] for _ in batch_size]
+
+        for step in range(self.max_length):
+            decoder_input = growing_beam[:, -1]
+            outputs = self.decoder(decoder_input, kwargs_decoder)
+            log_probabilities = torch.nn.functional.log_softmax(outputs[1])
+            vocab_size = log_probabilities.size(-1)
+
+            # The batch size changes as some beams finish so we define:
+            _B = log_probabilities.size(0) // self.beam_size
+
+            # Multiply each beam probability with the probability of the
+            # next token (conditioned on the words in the beam).
+            log_probabilities += topk_log_probabilities.view(-1, 1)
+
+            # if the beam has not attained the minimum required length we
+            # make the end token arbitrarily unlikely.
+            if step < self.min_length:
+                log_probabilities[self.end_token_id] = -1e20
+
+            # Remove repeating tri-grams
+            if(self.args.block_trigram):
+                if(step + 1 > 3):
+                    for i in range(_B * self.beam_size):
+                        tokens = [t for t in growing_beam[i]]
+                        trigrams = [(tokens[i-1], tokens[i], tokens[i+1]) for i in range(1, len(words) - 1)]
+                        last_trigram = tuple(trigrams[-1])
+                        if last_trigram in trigrams[:-1]:
+                            log_probabilities[i] = -1e20
+
+            # Find the `beam_size` (previous_beam + token) combinations with
+            # the highest score
+            topk_log_probabilities, topk_ids = log_probabilities.topk(
+                log_probabilities.view(_B, self.beam_size * vocab_size),
+                self.beam_size,
+                dim=1
+            )
+
+            # Apply the length penalty. The +1 accounts for the [EOS] token
+            # that will be added if the beam ends.
+            length_penalty = ((5.0 + (step + 1)) / 6.0) ** self.alpha
+            topk_scores = topk_log_probabilities / length_penalty
+
+            # Retrieve the corresponding respective beam and token id
+            # topk_token_ids[i] will be added to topk_beam_ids[i]
+            topk_beam_ids = topk_ids.div(vocab_size)
+            topk_token_ids = topk_ids.fmod(vocab_size)
+
+            # Retrieve the row index of the surviving beams in the original
+            # view of the log_probabilities tensor
+            surviving_beams_rows = (
+                topk_beam_ids + beam_offset[:_B].view(-1, 1)
+            ).view(-1)
+
+            # Append the last predictions
+            growing_beam = torch.cat(
+                [
+                    growing_beam.index_select(0, surviving_beams_rows),
+                    topk_token_ids.view(-1, 1),
+                ],
+                1,
+            )
+
+            # Check if any of the beam searches has ended during this
+            # growth step. Also if top beam (most probable) has ended
+            # for one element of the batch.
+            is_finished = topk_token_ids.eq(self.end_token_id)
+            if step + 1 == self.max_length:
+                is_finished.fill_(1)
+            is_top_beam_finished = is_finished[:, 0].eq(1)
+
+            # Save the finished searches
+            if is_finished.any():
+                predictions = growing_beam.view(-1, self.beam_size, growing_beam.size(1))
+                for i in range(is_finished.size(0)):
+                    if is_top_beam_finished[i]:
+                        is_finished[i].fill_(1)
+                    finished_hyp = is_finished[i].nonzero().view(-1)
+
+                    # Store finished hypotheses for this batch.
+                    b = batch_offset[i]
+                    for j in finished_hyp:
+                        hypotheses[b].append((topk_scores[i, j], predictions[i, j, :]))
+
+                    # If the batch reached the end, save the best hypotheses
+                    # in terms of length-penalized score.
+                    if is_top_beam_finished[i]:
+                        best_hyp = sorted(
+                            hypotheses[b], key=lambda x: x[0], reverse=True
+                        )
+                        best_score, best_prediction = best_hyp[0]
+                        results["scores"][b].append(best_score)
+                        results["predictions"][b].append(best_prediction)
+
+                non_finished = is_top_beam_finished.eq(0).nonzero().view(-1)
+                if len(non_finished) == 0:
+                    break
+
+                # Remove finished batches for the next step.
+                topk_log_probabilities = topk_log_probabilities.index_select(0, non_finished)
+                batch_offset = batch_offset.index_select(0, non_finished)
+                growing_beam = predictions.index_select(0, non_finished).view(
+                    -1, growing_beam.size(-1)
+                )
+
+            # Re-order the state for the next pass
+            surviving_beams_rows = surviving_beams_rows.index_select(0, non_finished)
+            kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[
+                "encoder_hidden_states"
+            ].index_select(0, surviving_beams_rows)
+
+        return results
+
+
+def tile(x, count, dim=0):
+    """
+    Tiles `x` along dimension `dim` `count` times.
+
+    Example:
+        >> ex = torch.tensor([1,2],[3,4])
+        >> tile(ex, 2, 0)
+        torch.Tensor([[1,2],[1,2],[3,4],[3,4]])
+    """
+    perm = list(range(len(x.size())))
+    if dim != 0:
+        perm[0], perm[dim] = perm[dim], perm[0]
+        x = x.permute(perm).contiguous()
+    out_size = list(x.size())
+    out_size[0] *= count
+    batch = x.size(0)
+    x = (
+        x.view(batch, -1)
+        .transpose(0, 1)
+        .repeat(count, 1)
+        .transpose(0, 1)
+        .contiguous()
+        .view(*out_size)
+    )
+    if dim != 0:
+        x = x.permute(perm).contiguous()
+    return x
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index d10f32c1fa..93f3c7e1f1 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -646,7 +646,7 @@ class BertModel(BertPreTrainedModel):
         if attention_mask.dim() == 2:
             if self.config.is_decoder:
                 batch_size, seq_length = input_ids.size()
-                seq_ids = torch.arange(seq_length)
+                seq_ids = torch.arange(seq_length, device=input_ids.device)
                 causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
                 extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
             else:
@@ -660,6 +660,13 @@ class BertModel(BertPreTrainedModel):
         extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
         extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
 
+        # If a 2D encoder attention mask is provided for the cross-attention
+        # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
+        if encoder_attention_mask is not None:
+            encoder_attention_mask = encoder_attention_mask[:, None, None, :]
+            encoder_attention_mask = encoder_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+            encoder_attention_mask = (1.0 - encoder_attention_mask) * -10000.0
+
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
         # attention_probs has shape bsz x n_heads x N x N
@@ -819,7 +826,7 @@ class BertForMaskedLM(BertPreTrainedModel):
                                    self.bert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None, lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None):
+                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, lm_labels=None, ):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
@@ -838,11 +845,8 @@ class BertForMaskedLM(BertPreTrainedModel):
         # 1. If a tensor that contains the indices of masked labels is provided,
         #    the cross-entropy is the MLM cross-entropy that measures the likelihood
         #    of predictions for masked words.
-        # 2. If encoder hidden states are provided we are in a causal situation where we
+        # 2. If `lm_label` is provided we are in a causal scenario where we
         #    try to predict the next word for each input in the encoder.
-        if masked_lm_labels is not None and lm_labels is not None:
-            raise AttributeError("Masked LM training with an encoder-decoder is not supported.")
-
         if masked_lm_labels is not None:
             loss_fct = CrossEntropyLoss(ignore_index=-1)  # -1 index = padding token
             masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
@@ -851,9 +855,9 @@ class BertForMaskedLM(BertPreTrainedModel):
         if lm_labels is not None:
             # we are doing next-token prediction; shift prediction scores and input ids by one
             prediction_scores = prediction_scores[:, :-1, :]
-            lm_labels = lm_labels[:, 1:, :]
+            lm_labels = lm_labels[:, 1:]
             loss_fct = CrossEntropyLoss(ignore_index=-1)
-            seq2seq_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), lm_labels.view(-1))
+            seq2seq_loss = loss_fct(prediction_scores.reshape(-1, self.config.vocab_size), lm_labels.reshape(-1))
             outputs = (seq2seq_loss,) + outputs
 
         return outputs  # (mlm_or_seq2seq_loss), prediction_scores, (hidden_states), (attentions)
diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 108fdaa853..2767dd2cd1 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -17,13 +17,12 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import os
 
 import torch
 from torch import nn
 
-from .file_utils import add_start_docstrings
 from .modeling_auto import AutoModel, AutoModelWithLMHead
-from .modeling_utils import PreTrainedModel, SequenceSummary
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +42,13 @@ class PreTrainedSeq2seq(nn.Module):
         self.decoder = decoder
 
     @classmethod
-    def from_pretrained(cls, encoder_pretrained_model_name_or_path=None, decoder_pretrained_model_name_or_path=None, *model_args, **kwargs):
+    def from_pretrained(
+        cls,
+        encoder_pretrained_model_name_or_path=None,
+        decoder_pretrained_model_name_or_path=None,
+        *model_args,
+        **kwargs
+    ):
         r""" Instantiates an encoder and a decoder from one or two base classes
         of the library from pre-trained model checkpoints.
 
@@ -108,23 +113,28 @@ class PreTrainedSeq2seq(nn.Module):
 
         # Separate the encoder- and decoder- specific kwargs. A kwarg is
         # decoder-specific it the key starts with `decoder_`
-        kwargs_decoder = {}
-        kwargs_encoder = kwargs
-        for key in kwargs_encoder.keys():
-            if key.startswith("decoder_"):
-                kwargs_decoder[key.replace("decoder_", "")] = kwargs_encoder.pop(key)
+        kwargs_encoder = {
+            argument: value
+            for argument, value in kwargs.items()
+            if not argument.startswith("decoder_")
+        }
+        kwargs_decoder = {
+            argument[len("decoder_") :]: value
+            for argument, value in kwargs.items()
+            if argument.startswith("decoder_")
+        }
 
         # Load and initialize the encoder and decoder
-        #  The distinction between encoder and decoder at the model level is made
-        #  by the value of the flag `is_decoder` that we need to set correctly.
-        encoder = kwargs.pop("encoder_model", None)
+        # The distinction between encoder and decoder at the model level is made
+        # by the value of the flag `is_decoder` that we need to set correctly.
+        encoder = kwargs_encoder.pop("encoder_model", None)
         if encoder is None:
             kwargs_encoder["is_decoder"] = False
             encoder = AutoModel.from_pretrained(
                 encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder
             )
 
-        decoder = kwargs.pop("decoder_model", None)
+        decoder = kwargs_decoder.pop("model", None)
         if decoder is None:
             kwargs_decoder["is_decoder"] = True
             decoder = AutoModelWithLMHead.from_pretrained(
@@ -135,6 +145,12 @@ class PreTrainedSeq2seq(nn.Module):
 
         return model
 
+    def save_pretrained(self, save_directory):
+        """ Save a Seq2Seq model and its configuration file in a format
+        such that it can be loaded using `:func:`~transformers.PreTrainedSeq2seq.from_pretrained` """
+        self.encoder.save_pretrained(os.path.join(save_directory, "encoder"))
+        self.decoder.save_pretrained(os.path.join(save_directory, "decoder"))
+
     def forward(self, encoder_input_ids, decoder_input_ids, **kwargs):
         """ The forward pass on a seq2eq depends what we are performing:
 
@@ -155,22 +171,29 @@ class PreTrainedSeq2seq(nn.Module):
         """
         # Separate the encoder- and decoder- specific kwargs. A kwarg is
         # decoder-specific it the key starts with `decoder_`
-        kwargs_decoder = {}
-        kwargs_encoder = kwargs
-        for key in kwargs_encoder.keys():
-            if key.startswith("decoder_"):
-                kwargs_decoder[key.replace("decoder_", "")] = kwargs_encoder.pop(key)
+        kwargs_encoder = {
+            argument: value
+            for argument, value in kwargs.items()
+            if not argument.startswith("decoder_")
+        }
+        kwargs_decoder = {
+            argument[len("decoder_") :]: value
+            for argument, value in kwargs.items()
+            if argument.startswith("decoder_")
+        }
 
         # Encode if needed (training, first prediction pass)
         encoder_hidden_states = kwargs_encoder.pop("encoder_hidden_states", None)
         if encoder_hidden_states is None:
             encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder)
-            encoder_hidden_states = encoder_outputs[0][-1]  # output of the encoder *stack*
+            encoder_hidden_states = encoder_outputs[0][
+                -1
+            ]  # output of the encoder *stack*
         else:
             encoder_outputs = ()
 
         # Decode
-        kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states
+        kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states[None, :, :]
         decoder_outputs = self.decoder(decoder_input_ids, **kwargs_decoder)
 
         return decoder_outputs + encoder_outputs
@@ -201,9 +224,25 @@ class Model2Model(PreTrainedSeq2seq):
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
-        model = super(Model2Model, cls).from_pretrained(encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
-                                                        decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
-                                                        **kwargs)
+
+        if (
+            "bert" not in pretrained_model_name_or_path
+            or "roberta" in pretrained_model_name_or_path
+            or "distilbert" in pretrained_model_name_or_path
+        ):
+            raise ValueError("Only the Bert model is currently supported.")
+
+        model = super(Model2Model, cls).from_pretrained(
+            encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
+            decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
+            **kwargs
+        )
+
+        # Some architectures require for the decoder to be initialized randomly
+        # before fine-tuning.
+        if kwargs.get("decoder_initialize_randomly", False):
+            model.decoder.init_weights()
+
         return model
 
 

From 438f2730a03e19bc21f2823c659ceaed0dfe8ef7 Mon Sep 17 00:00:00 2001
From: altsoph <altsoph@gmail.com>
Date: Fri, 25 Oct 2019 13:22:58 +0300
Subject: [PATCH 097/269] Evaluation code fixed.

---
 examples/run_lm_finetuning.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 571bcb4391..4d32385e40 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -86,6 +86,7 @@ class TextDataset(Dataset):
             # Note that we are loosing the last truncated example here for the sake of simplicity (no padding)
             # If your dataset is small, first you should loook for a bigger one :-) and second you
             # can change this behavior by adding (model specific) padding.
+            self.examples.append(tokenizer.build_inputs_with_special_tokens(tokenized_text[-block_size:])) # DIRTY!
 
             logger.info("Saving features into cached file %s", cached_features_file)
             with open(cached_features_file, 'wb') as handle:
@@ -309,10 +310,12 @@ def evaluate(args, model, tokenizer, prefix=""):
     model.eval()
 
     for batch in tqdm(eval_dataloader, desc="Evaluating"):
-        batch = batch.to(args.device)
+        inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch)
+        inputs = inputs.to(args.device)
+        labels = labels.to(args.device)
 
         with torch.no_grad():
-            outputs = model(batch, masked_lm_labels=batch) if args.mlm else model(batch, labels=batch)
+            outputs = model(inputs, masked_lm_labels=labels) if args.mlm else model(inputs, labels=labels)
             lm_loss = outputs[0]
             eval_loss += lm_loss.mean().item()
         nb_eval_steps += 1
@@ -540,4 +543,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
\ No newline at end of file

From 079bfb32fba4f2b39d344ca7af88d79a3ff27c7c Mon Sep 17 00:00:00 2001
From: altsoph <altsoph@gmail.com>
Date: Fri, 25 Oct 2019 13:26:37 +0300
Subject: [PATCH 098/269] Evaluation fixed.

---
 examples/run_lm_finetuning.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 4d32385e40..982d8aa1b7 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -86,7 +86,6 @@ class TextDataset(Dataset):
             # Note that we are loosing the last truncated example here for the sake of simplicity (no padding)
             # If your dataset is small, first you should loook for a bigger one :-) and second you
             # can change this behavior by adding (model specific) padding.
-            self.examples.append(tokenizer.build_inputs_with_special_tokens(tokenized_text[-block_size:])) # DIRTY!
 
             logger.info("Saving features into cached file %s", cached_features_file)
             with open(cached_features_file, 'wb') as handle:
@@ -543,4 +542,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()

From dfce40969141eb037e8af3ed64e490a876386bf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Tue, 29 Oct 2019 17:10:20 +0100
Subject: [PATCH 099/269] resolve PR comments

---
 examples/run_summarization_finetuning.py      | 292 +++++-----------
 examples/run_summarization_finetuning_test.py |  76 ----
 examples/utils_summarization.py               | 184 ++++++++++
 examples/utils_summarization_test.py          | 133 +++++++
 transformers/modeling_beam_search.py          | 325 ++++++++++--------
 transformers/modeling_bert.py                 |  31 +-
 transformers/modeling_seq2seq.py              |  79 +++--
 7 files changed, 647 insertions(+), 473 deletions(-)
 delete mode 100644 examples/run_summarization_finetuning_test.py
 create mode 100644 examples/utils_summarization.py
 create mode 100644 examples/utils_summarization_test.py

diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py
index 64bee82c5b..1888f56caf 100644
--- a/examples/run_summarization_finetuning.py
+++ b/examples/run_summarization_finetuning.py
@@ -16,10 +16,9 @@
 """ Finetuning seq2seq models for sequence generation."""
 
 import argparse
-from collections import deque
+import functools
 import logging
 import os
-import pickle
 import random
 import sys
 
@@ -29,7 +28,22 @@ import torch
 from torch.optim import Adam
 from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
 
-from transformers import AutoTokenizer, PreTrainedSeq2seq, Model2Model
+from transformers import (
+    AutoTokenizer,
+    BertForMaskedLM,
+    BertConfig,
+    PreTrainedSeq2seq,
+    Model2Model,
+)
+
+from utils_summarization import (
+    CNNDailyMailDataset,
+    encode_for_summarization,
+    fit_to_block_size,
+    build_lm_labels,
+    build_mask,
+    compute_token_type_ids,
+)
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
@@ -46,194 +60,41 @@ def set_seed(args):
 # ------------
 
 
-class TextDataset(Dataset):
-    """ Abstracts the dataset used to train seq2seq models.
-
-    CNN/Daily News:
-
-    The CNN/Daily News raw datasets are downloaded from [1]. The stories are
-    stored in different files; the summary appears at the end of the story as
-    sentences that are prefixed by the special `@highlight` line. To process
-    the data, untar both datasets in the same folder, and pass the path to this
-    folder as the "data_dir argument. The formatting code was inspired by [2].
-
-    [1] https://cs.nyu.edu/~kcho/
-    [2] https://github.com/abisee/cnn-dailymail/
-    """
-
-    def __init__(self, tokenizer, prefix="train", data_dir="", block_size=512):
-        assert os.path.isdir(data_dir)
-
-        # Load the features that have already been computed, if any
-        cached_features_file = os.path.join(
-            data_dir, "cached_lm_{}_{}".format(block_size, prefix)
-        )
-        if os.path.exists(cached_features_file):
-            logger.info("Loading features from cached file %s", cached_features_file)
-            with open(cached_features_file, "rb") as source:
-                self.examples = pickle.load(source)
-                return
-
-        logger.info("Creating features from dataset at %s", data_dir)
-        datasets = ["cnn", "dailymail"]
-
-        self.examples = {"source": [], "target": []}
-        for dataset in datasets:
-            path_to_stories = os.path.join(data_dir, dataset, "stories")
-            story_filenames_list = os.listdir(path_to_stories)
-            for story_filename in story_filenames_list:
-                path_to_story = os.path.join(path_to_stories, story_filename)
-                if not os.path.isfile(path_to_story):
-                    continue
-
-                with open(path_to_story, encoding="utf-8") as source:
-                    raw_story = source.read()
-                    story_lines, summary_lines = process_story(raw_story)
-                    if len(summary_lines) == 0 or len(story_lines) == 0:
-                        continue
-
-                story_token_ids, summary_token_ids = _encode_for_summarization(
-                    story_lines, summary_lines, tokenizer
-                )
-                story_seq = _fit_to_block_size(story_token_ids, block_size)
-                self.examples["source"].append(story_seq)
-
-                summary_seq = _fit_to_block_size(summary_token_ids, block_size)
-                self.examples["summary"].append(summary_seq)
-
-        logger.info("Saving features into cache file %s", cached_features_file)
-        with open(cached_features_file, "wb") as sink:
-            pickle.dump(self.examples, sink, protocol=pickle.HIGHEST_PROTOCOL)
-
-    def __len__(self):
-        return len(self.examples)
-
-    def __getitem__(self, items):
-        return (
-            torch.tensor(self.examples["source"][items]),
-            torch.tensor(self.examples["target"][items]),
-        )
-
-
-def process_story(raw_story):
-    """ Extract the story and summary from a story file.
-
-    Attributes:
-        raw_story (str): content of the story file as an utf-8 encoded string.
-
-    Raises:
-        IndexError: If the stoy is empty or contains no highlights.
-    """
-    nonempty_lines = list(
-        filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")])
-    )
-
-    # for some unknown reason some lines miss a period, add it
-    nonempty_lines = [_add_missing_period(line) for line in nonempty_lines]
-
-    # gather article lines
-    story_lines = []
-    lines = deque(nonempty_lines)
-    while True:
-        try:
-            element = lines.popleft()
-            if element.startswith("@highlight"):
-                break
-            story_lines.append(element)
-        except IndexError:
-            # if "@highlight" is absent from the file we pop
-            # all elements until there is None.
-            return story_lines, []
-
-    # gather summary lines
-    summary_lines = list(filter(lambda t: not t.startswith("@highlight"), lines))
-
-    return story_lines, summary_lines
-
-
-def _encode_for_summarization(story_lines, summary_lines, tokenizer):
-    """ Encode the story and summary lines, and join them
-    as specified in [1] by using `[SEP] [CLS]` tokens to separate
-    sentences.
-    """
-    story_lines_token_ids = [
-        tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line))
-        for line in story_lines
-    ]
-    summary_lines_token_ids = [
-        tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line))
-        for line in summary_lines
-    ]
-
-    story_token_ids = [
-        token for sentence in story_lines_token_ids for token in sentence
-    ]
-    summary_token_ids = [
-        token for sentence in summary_lines_token_ids for token in sentence
-    ]
-
-    return story_token_ids, summary_token_ids
-
-
-def _add_missing_period(line):
-    END_TOKENS = [".", "!", "?", "...", "'", "`", '"', u"\u2019", u"\u2019", ")"]
-    if line.startswith("@highlight"):
-        return line
-    if line[-1] in END_TOKENS:
-        return line
-    return line + "."
-
-
-def _fit_to_block_size(sequence, block_size):
-    """ Adapt the source and target sequences' lengths to the block size.
-    If the sequence is shorter than the block size we pad it with -1 ids
-    which correspond to padding tokens.
-    """
-    if len(sequence) > block_size:
-        return sequence[:block_size]
-    else:
-        sequence.extend([0] * (block_size - len(sequence)))
-        return sequence
-
-
-def mask_padding_tokens(sequence):
-    """ Padding token, encoded as 0, are represented by the value -1 in the
-    masks """
-    padded = sequence.clone()
-    padded[padded == 0] = -1
-    return padded
-
-
 def load_and_cache_examples(args, tokenizer):
-    dataset = TextDataset(tokenizer, data_dir=args.data_dir)
+    dataset = CNNDailyMailDataset(tokenizer, data_dir=args.data_dir)
     return dataset
 
 
-def compute_token_type_ids(batch, separator_token_id):
-    """ Segment embeddings as described in [1]
+def collate(data, tokenizer, block_size):
+    """ List of tuple as an input. """
+    # remove the files with empty an story/summary, encode and fit to block
+    data = filter(lambda x: not (len(x[0]) == 0 or len(x[1]) == 0), data)
+    data = [
+        encode_for_summarization(story, summary, tokenizer) for story, summary in data
+    ]
+    data = [
+        (
+            fit_to_block_size(story, block_size, tokenizer.pad_token_id),
+            fit_to_block_size(summary, block_size, tokenizer.pad_token_id),
+        )
+        for story, summary in data
+    ]
 
-    The values {0,1} were found in the repository [2].
+    stories = torch.tensor([story for story, summary in data])
+    summaries = torch.tensor([summary for story, summary in data])
+    encoder_token_type_ids = compute_token_type_ids(stories, tokenizer.cls_token_id)
+    encoder_mask = build_mask(stories, tokenizer.pad_token_id)
+    decoder_mask = build_mask(summaries, tokenizer.pad_token_id)
+    lm_labels = build_lm_labels(summaries, tokenizer.pad_token_id)
 
-    Attributes:
-        batch: torch.Tensor, size [batch_size, block_size]
-            Batch of input.
-        separator_token_id: int
-            The value of the token that separates the segments.
-
-    [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders."
-        arXiv preprint arXiv:1908.08345 (2019).
-    [2] https://github.com/nlpyang/PreSumm (/src/prepro/data_builder.py, commit fac1217)
-    """
-    batch_embeddings = []
-    sentence_num = 0
-    for sequence in batch:
-        embeddings = []
-        for s in sequence:
-            if s == separator_token_id:
-                sentence_num += 1
-            embeddings.append(sentence_num % 2)
-        batch_embeddings.append(embeddings)
-    return torch.tensor(batch_embeddings)
+    return (
+        stories,
+        summaries,
+        encoder_token_type_ids,
+        encoder_mask,
+        decoder_mask,
+        lm_labels,
+    )
 
 
 # ----------
@@ -252,7 +113,7 @@ class BertSumOptimizer(object):
         arXiv preprint arXiv:1908.08345 (2019).
     """
 
-    def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-9):
+    def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-8):
         self.encoder = model.encoder
         self.decoder = model.decoder
         self.lr = lr
@@ -306,8 +167,12 @@ def train(args, model, tokenizer):
     args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
     train_dataset = load_and_cache_examples(args, tokenizer)
     train_sampler = RandomSampler(train_dataset)
+    model_collate_fn = functools.partial(collate, tokenizer=tokenizer, block_size=512)
     train_dataloader = DataLoader(
-        train_dataset, sampler=train_sampler, batch_size=args.train_batch_size
+        train_dataset,
+        sampler=train_sampler,
+        batch_size=args.train_batch_size,
+        collate_fn=model_collate_fn,
     )
 
     # Training schedule
@@ -351,26 +216,23 @@ def train(args, model, tokenizer):
     for _ in train_iterator:
         epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=True)
         for step, batch in enumerate(epoch_iterator):
-            source, target = batch
-            token_type_ids = compute_token_type_ids(source, tokenizer.cls_token_id)
-            labels_src = mask_padding_tokens(source)
-            labels_tgt = mask_padding_tokens(target)
+            source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch
 
             source = source.to(args.device)
             target = target.to(args.device)
-            token_type_ids = token_type_ids.to(args.device)
-            labels_src = labels_src.to(args.device)
-            labels_tgt = labels_tgt.to(args.device)
+            encoder_token_type_ids = encoder_token_type_ids.to(args.device)
+            encoder_mask = encoder_mask.to(args.device)
+            decoder_mask = decoder_mask.to(args.device)
+            lm_labels = lm_labels.to(args.device)
 
             model.train()
             outputs = model(
                 source,
                 target,
-                token_type_ids=token_type_ids,
-                decoder_encoder_attention_mask=labels_src,
-                decoder_attention_mask=labels_tgt,
-                decoder_lm_labels=labels_tgt,
-                decoder_initialize_randomly=True,
+                encoder_token_type_ids=encoder_token_type_ids,
+                encoder_attention_mask=encoder_mask,
+                decoder_attention_mask=decoder_mask,
+                decoder_lm_labels=lm_labels,
             )
 
             loss = outputs[0]
@@ -421,21 +283,23 @@ def evaluate(args, model, tokenizer, prefix=""):
     model.eval()
 
     for batch in tqdm(eval_dataloader, desc="Evaluating"):
-        source, target = batch
-        labels_src = mask_padding_tokens(source)
-        labels_tgt = mask_padding_tokens(target)
-        source.to(args.device)
-        target.to(args.device)
-        labels_src.to(args.device)
-        labels_tgt.to(args.device)
+        source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch
+
+        source = source.to(args.device)
+        target = target.to(args.device)
+        encoder_token_type_ids = encoder_token_type_ids.to(args.device)
+        encoder_mask = encoder_mask.to(args.device)
+        decoder_mask = decoder_mask.to(args.device)
+        lm_labels = lm_labels.to(args.device)
 
         with torch.no_grad():
             outputs = model(
                 source,
                 target,
-                decoder_encoder_attention_mask=labels_src,
-                decoder_attention_mask=labels_tgt,
-                decoder_lm_labels=labels_tgt,
+                encoder_token_type_ids=encoder_token_type_ids,
+                encoder_attention_mask=encoder_mask,
+                decoder_attention_mask=decoder_mask,
+                decoder_lm_labels=lm_labels,
             )
             lm_loss = outputs[0]
             eval_loss += lm_loss.mean().item()
@@ -525,7 +389,7 @@ def main():
     )
     parser.add_argument(
         "--num_train_epochs",
-        default=1,
+        default=10,
         type=int,
         help="Total number of training epochs to perform.",
     )
@@ -558,9 +422,13 @@ def main():
         args.device = torch.device("cuda")
         args.n_gpu = torch.cuda.device_count()
 
-    # Load pretrained model and tokenizer
+    # Load pretrained model and tokenizer. The decoder's weights are randomly initialized.
     tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
-    model = Model2Model.from_pretrained(args.model_name_or_path)
+    config = BertConfig.from_pretrained(args.model_name_or_path)
+    decoder_model = BertForMaskedLM(config)
+    model = Model2Model.from_pretrained(
+        args.model_name_or_path, decoder_model=decoder_model
+    )
 
     # Setup logging
     logging.basicConfig(
diff --git a/examples/run_summarization_finetuning_test.py b/examples/run_summarization_finetuning_test.py
deleted file mode 100644
index fd997ee0c2..0000000000
--- a/examples/run_summarization_finetuning_test.py
+++ /dev/null
@@ -1,76 +0,0 @@
-# coding=utf-8
-# Copyright 2019 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import unittest
-
-from run_summarization_finetuning import _fit_to_block_size, process_story
-
-
-class DataLoaderTest(unittest.TestCase):
-    def setUp(self):
-        self.block_size = 10
-
-    def test_truncate_sequence_too_small(self):
-        """ Pad the sequence with 0 if the sequence is smaller than the block size."""
-        sequence = [1, 2, 3, 4]
-        expected_output = [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
-        self.assertEqual(_fit_to_block_size(sequence, self.block_size), expected_output)
-
-    def test_truncate_sequence_fit_exactly(self):
-        sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        self.assertEqual(_fit_to_block_size(sequence, self.block_size), expected_output)
-
-    def test_truncate_sequence_too_big(self):
-        sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
-        expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        self.assertEqual(_fit_to_block_size(sequence, self.block_size), expected_output)
-
-    def test_process_story_no_highlights(self):
-        """ Processing a story with no highlights should raise an exception.
-        """
-        raw_story = """It was the year of Our Lord one thousand seven hundred and
-        seventy-five.\n\nSpiritual revelations were conceded to England at that
-        favoured period, as at this."""
-        _, summary = process_story(raw_story)
-        self.assertEqual(summary, [])
-
-    def test_process_empty_story(self):
-        """ An empty story should also raise and exception.
-        """
-        raw_story = ""
-        story, summary = process_story(raw_story)
-        self.assertEqual(story, [])
-        self.assertEqual(summary, [])
-
-    def test_story_with_missing_period(self):
-        raw_story = (
-            "It was the year of Our Lord one thousand seven hundred and "
-            "seventy-five\n\nSpiritual revelations were conceded to England "
-            "at that favoured period, as at this.\n@highlight\n\nIt was the best of times"
-        )
-        story_lines, summary_lines = process_story(raw_story)
-
-        expected_story_lines = [
-            "It was the year of Our Lord one thousand seven hundred and seventy-five.",
-            "Spiritual revelations were conceded to England at that favoured period, as at this.",
-        ]
-        self.assertEqual(expected_story_lines, story_lines)
-
-        expected_summary_lines = ["It was the best of times."]
-        self.assertEqual(expected_summary_lines, summary_lines)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py
new file mode 100644
index 0000000000..cd8bc4bc2b
--- /dev/null
+++ b/examples/utils_summarization.py
@@ -0,0 +1,184 @@
+from collections import deque
+import os
+
+import torch
+from torch.utils.data import Dataset
+
+
+# ------------
+# Data loading
+# ------------
+
+
+class CNNDailyMailDataset(Dataset):
+    """ Abstracts the dataset used to train seq2seq models.
+
+    CNN/Daily News:
+
+    The CNN/Daily News raw datasets are downloaded from [1]. The stories are
+    stored in different files; the summary appears at the end of the story as
+    sentences that are prefixed by the special `@highlight` line. To process
+    the data, untar both datasets in the same folder, and pass the path to this
+    folder as the "data_dir argument. The formatting code was inspired by [2].
+
+    [1] https://cs.nyu.edu/~kcho/
+    [2] https://github.com/abisee/cnn-dailymail/
+    """
+
+    def __init__(self, tokenizer, prefix="train", data_dir=""):
+        assert os.path.isdir(data_dir)
+        self.tokenizer = tokenizer
+
+        # We initialize the class by listing all the files that contain
+        # stories and summaries. Files are not read in memory given
+        # the size of the corpus.
+        self.stories_path = []
+        datasets = ("cnn", "dailymail")
+        for dataset in datasets:
+            path_to_stories = os.path.join(data_dir, dataset, "stories")
+            story_filenames_list = os.listdir(path_to_stories)
+            for story_filename in story_filenames_list:
+                path_to_story = os.path.join(path_to_stories, story_filename)
+                if not os.path.isfile(path_to_story):
+                    continue
+                self.stories_path.append(path_to_story)
+
+    def __len__(self):
+        return len(self.stories_path)
+
+    def __getitem__(self, idx):
+        story_path = self.stories_path[idx]
+        with open(story_path, encoding="utf-8") as source:
+            raw_story = source.read()
+            story_lines, summary_lines = process_story(raw_story)
+        return story_lines, summary_lines
+
+
+def process_story(raw_story):
+    """ Extract the story and summary from a story file.
+
+    Attributes:
+        raw_story (str): content of the story file as an utf-8 encoded string.
+
+    Raises:
+        IndexError: If the stoy is empty or contains no highlights.
+    """
+    nonempty_lines = list(
+        filter(lambda x: len(x) != 0, [line.strip() for line in raw_story.split("\n")])
+    )
+
+    # for some unknown reason some lines miss a period, add it
+    nonempty_lines = [_add_missing_period(line) for line in nonempty_lines]
+
+    # gather article lines
+    story_lines = []
+    lines = deque(nonempty_lines)
+    while True:
+        try:
+            element = lines.popleft()
+            if element.startswith("@highlight"):
+                break
+            story_lines.append(element)
+        except IndexError:
+            # if "@highlight" is absent from the file we pop
+            # all elements until there is None.
+            return story_lines, []
+
+    # gather summary lines
+    summary_lines = list(filter(lambda t: not t.startswith("@highlight"), lines))
+
+    return story_lines, summary_lines
+
+
+def _add_missing_period(line):
+    END_TOKENS = [".", "!", "?", "...", "'", "`", '"', u"\u2019", u"\u2019", ")"]
+    if line.startswith("@highlight"):
+        return line
+    if line[-1] in END_TOKENS:
+        return line
+    return line + "."
+
+
+# --------------------------
+# Encoding and preprocessing
+# --------------------------
+
+
+def fit_to_block_size(sequence, block_size, pad_token):
+    """ Adapt the source and target sequences' lengths to the block size.
+    If the sequence is shorter than the block size we pad it with -1 ids
+    which correspond to padding tokens.
+    """
+    if len(sequence) > block_size:
+        return sequence[:block_size]
+    else:
+        sequence.extend([pad_token] * (block_size - len(sequence)))
+        return sequence
+
+
+def build_lm_labels(sequence, pad_token):
+    """ Padding token, encoded as 0, are represented by the value -1 so they
+    are not taken into account in the loss computation. """
+    padded = sequence.clone()
+    padded[padded == pad_token] = -1
+    return padded
+
+
+def build_mask(sequence, pad_token):
+    """ Builds the mask. The attention mechanism will only attend to positions
+    with value 1. """
+    mask = sequence.clone()
+    mask[mask != pad_token] = 1
+    mask[mask == pad_token] = 0
+    return mask
+
+
+def encode_for_summarization(story_lines, summary_lines, tokenizer):
+    """ Encode the story and summary lines, and join them
+    as specified in [1] by using `[SEP] [CLS]` tokens to separate
+    sentences.
+    """
+    story_lines_token_ids = [
+        tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line))
+        for line in story_lines
+    ]
+    summary_lines_token_ids = [
+        tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line))
+        for line in summary_lines
+    ]
+
+    story_token_ids = [
+        token for sentence in story_lines_token_ids for token in sentence
+    ]
+    summary_token_ids = [
+        token for sentence in summary_lines_token_ids for token in sentence
+    ]
+
+    return story_token_ids, summary_token_ids
+
+
+def compute_token_type_ids(batch, separator_token_id):
+    """ Segment embeddings as described in [1]
+
+    The values {0,1} were found in the repository [2].
+
+    Attributes:
+        batch: torch.Tensor, size [batch_size, block_size]
+            Batch of input.
+        separator_token_id: int
+            The value of the token that separates the segments.
+
+    [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders."
+        arXiv preprint arXiv:1908.08345 (2019).
+    [2] https://github.com/nlpyang/PreSumm (/src/prepro/data_builder.py, commit fac1217)
+    """
+    batch_embeddings = []
+    for sequence in batch:
+        sentence_num = 0
+        embeddings = []
+        for s in sequence:
+            if s == separator_token_id:
+                sentence_num += 1
+            embeddings.append(sentence_num % 2)
+        batch_embeddings.append(embeddings)
+    return torch.tensor(batch_embeddings)
diff --git a/examples/utils_summarization_test.py b/examples/utils_summarization_test.py
new file mode 100644
index 0000000000..7a02f8fa1f
--- /dev/null
+++ b/examples/utils_summarization_test.py
@@ -0,0 +1,133 @@
+# coding=utf-8
+# Copyright 2019 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+import numpy as np
+import torch
+
+from utils_summarization import (
+    compute_token_type_ids,
+    fit_to_block_size,
+    build_mask,
+    build_lm_labels,
+    process_story,
+)
+
+
+class SummarizationDataProcessingTest(unittest.TestCase):
+    def setUp(self):
+        self.block_size = 10
+
+    def test_fit_to_block_sequence_too_small(self):
+        """ Pad the sequence with 0 if the sequence is smaller than the block size."""
+        sequence = [1, 2, 3, 4]
+        expected_output = [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
+        self.assertEqual(
+            fit_to_block_size(sequence, self.block_size, 0), expected_output
+        )
+
+    def test_fit_to_block_sequence_fit_exactly(self):
+        """ Do nothing if the sequence is the right size. """
+        sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        self.assertEqual(
+            fit_to_block_size(sequence, self.block_size, 0), expected_output
+        )
+
+    def test_fit_to_block_sequence_too_big(self):
+        """ Truncate the sequence if it is too long. """
+        sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+        expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        self.assertEqual(
+            fit_to_block_size(sequence, self.block_size, 0), expected_output
+        )
+
+    def test_process_story_no_highlights(self):
+        """ Processing a story with no highlights returns an empty list for the summary.
+        """
+        raw_story = """It was the year of Our Lord one thousand seven hundred and
+        seventy-five.\n\nSpiritual revelations were conceded to England at that
+        favoured period, as at this."""
+        _, summary_lines = process_story(raw_story)
+        self.assertEqual(summary_lines, [])
+
+    def test_process_empty_story(self):
+        """ An empty story returns an empty collection of lines.
+        """
+        raw_story = ""
+        story_lines, summary_lines = process_story(raw_story)
+        self.assertEqual(story_lines, [])
+        self.assertEqual(summary_lines, [])
+
+    def test_process_story_with_missing_period(self):
+        raw_story = (
+            "It was the year of Our Lord one thousand seven hundred and "
+            "seventy-five\n\nSpiritual revelations were conceded to England "
+            "at that favoured period, as at this.\n@highlight\n\nIt was the best of times"
+        )
+        story_lines, summary_lines = process_story(raw_story)
+
+        expected_story_lines = [
+            "It was the year of Our Lord one thousand seven hundred and seventy-five.",
+            "Spiritual revelations were conceded to England at that favoured period, as at this.",
+        ]
+        self.assertEqual(expected_story_lines, story_lines)
+
+        expected_summary_lines = ["It was the best of times."]
+        self.assertEqual(expected_summary_lines, summary_lines)
+
+    def test_build_lm_labels_no_padding(self):
+        sequence = torch.tensor([1, 2, 3, 4])
+        expected = sequence
+        np.testing.assert_array_equal(
+            build_lm_labels(sequence, 0).numpy(), expected.numpy()
+        )
+
+    def test_build_lm_labels(self):
+        sequence = torch.tensor([1, 2, 3, 4, 0, 0, 0])
+        expected = torch.tensor([1, 2, 3, 4, -1, -1, -1])
+        np.testing.assert_array_equal(
+            build_lm_labels(sequence, 0).numpy(), expected.numpy()
+        )
+
+    def test_build_mask_no_padding(self):
+        sequence = torch.tensor([1, 2, 3, 4])
+        expected = torch.tensor([1, 1, 1, 1])
+        np.testing.assert_array_equal(
+            build_mask(sequence, 0).numpy(), expected.numpy()
+        )
+
+    def test_build_mask(self):
+        sequence = torch.tensor([1, 2, 3, 4, 23, 23, 23])
+        expected = torch.tensor([1, 1, 1, 1, 0, 0, 0])
+        np.testing.assert_array_equal(
+            build_mask(sequence, 23).numpy(), expected.numpy()
+        )
+
+    def test_compute_token_type_ids(self):
+        separator = 101
+        batch = torch.tensor(
+            [[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]]
+        )
+        expected = torch.tensor(
+            [[0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 1, 1], [0, 1, 1, 1, 0, 0]]
+        )
+
+        result = compute_token_type_ids(batch, separator)
+        np.testing.assert_array_equal(result, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/transformers/modeling_beam_search.py b/transformers/modeling_beam_search.py
index 3a27625f90..171dcb7247 100644
--- a/transformers/modeling_beam_search.py
+++ b/transformers/modeling_beam_search.py
@@ -26,189 +26,220 @@ import torch
 from torch import nn
 
 
-class ModelWithBeamSearch(nn.Module):
+class TransformerBeamSearch(nn.Module):
     def __init__(
         self,
         model,
+        tokenizer,
+        batch_size,
         beam_size,
-        start_token_id,
-        end_token_id,
-        pad_token_id,
         min_length,
         max_length,
-        alpha,
-        block_trigram=True,
+        alpha=0,
+        block_repeating_trigram=True,
     ):
         """
         Attributes:
             mask_word_id: token id that corresponds to the mask
         """
-        super(ModelWithBeamSearch, self).__init__()
+        super(TransformerBeamSearch, self).__init__()
         self.model = model
+        self.tokenizer = tokenizer
+
+        self.start_token_id = tokenizer.start_token_id
+        self.end_token_id = tokenizer.end_token_id
+        self.pad_token_id = tokenizer.pad_token_id
+
         self.beam_size = beam_size
-        self.start_token_id = start_token_id
-        self.end_token_id = end_token_id
-        self.pad_token_id = pad_token_id
         self.min_length = min_length
         self.max_length = max_length
-        self.alpha = alpha
-        self.block_trigram = block_trigram
 
-    def forward(self, input_ids, **kwargs):
-        # Separate the encoder- and decoder- specific kwargs. A kwarg is
-        # decoder-specific it the key starts with `decoder_`
+        self.block_repeating_trigram = block_repeating_trigram
+        self.apply_length_penalty = False if alpha == 0 else True
+        self.alpha = alpha
+
+        # State of the beam
+        self.hypotheses = [[] for _ in range(batch_size)]
+        self.batch_offset = torch.arange(batch_size, dtype=torch.long)
+        self.beam_offset = torch.arange(
+            0, batch_size * self.beam_size, step=self.beam_size, dtype=torch.long
+        )
+        self.growing_beam = torch.full(
+            (batch_size * self.beam_size, 1), self.start_token_id, dtype=torch.long
+        )
+        self.topk_log_probabilities = torch.tensor(
+            [0.0] + [float("-inf")] * (self.beam_size - 1), dtype=torch.float
+        ).repeat(batch_size)
+        self.results = {
+            "prediction": [[] for _ in batch_size],
+            "scores": [[] for _ in batch_size],
+        }
+        self._step = 0
+        self.is_done = False
+
+    def step(self, log_probabilities):
+        """ Grows the beam by one step. """
+        self._step += 1
+
+        # The batch size changes as some beams finish so we define _B
+        vocab_size = log_probabilities.size(-1)
+        _B = log_probabilities.size(0) // self.beam_size
+
+        # Multiply each beam probability with the probability of the
+        # next token (conditioned on the words in the beam).
+        log_probabilities += self.topk_log_probabilities.view(-1, 1)
+
+        self.enforce_min_length(log_probabilities)
+        if self.block_repeating_trigram:
+            self.remove_repeating_trigrams(log_probabilities, _B)
+
+        # Find the `beam_size` (previous_beam + token) combinations with
+        # the highest score
+        topk_log_probabilities, topk_ids = log_probabilities.topk(
+            log_probabilities.view(_B, self.beam_size * vocab_size),
+            self.beam_size,
+            dim=1,
+        )
+
+        # Apply the length penalty. The +1 accounts for the [EOS] token
+        # that will be added if the beam ends.
+        topk_scores = topk_log_probabilities / self.length_penalty()
+
+        # Retrieve the corresponding respective beam and token id
+        # topk_token_ids[i] will be added to topk_beam_ids[i]
+        topk_beam_ids = topk_ids.div(vocab_size)
+        topk_token_ids = topk_ids.fmod(vocab_size)
+
+        # Retrieve the row index of the surviving beams in the original
+        # view of the log_probabilities tensor
+        surviving_beams_rows = (topk_beam_ids + self.beam_offset[:_B].view(-1, 1)).view(
+            -1
+        )
+
+        # Append the last predictions
+        self.growing_beam = torch.cat(
+            [
+                self.growing_beam.index_select(0, surviving_beams_rows),
+                topk_token_ids.view(-1, 1),
+            ],
+            1,
+        )
+
+        # Check if any of the beam searches has ended during this
+        # growth step. Also if top beam (most probable) has ended
+        # for one element of the batch.
+        is_finished = topk_token_ids.eq(self.end_token_id)
+        self.enforce_max_length()
+        is_top_beam_finished = is_finished[:, 0].eq(1)
+
+        # Save the finished searches
+        if is_finished.any():
+            predictions = self.growing_beam.view(
+                -1, self.beam_size, self.growing_beam.size(1)
+            )
+            for i in range(is_finished.size(0)):
+                if is_top_beam_finished[i]:
+                    is_finished[i].fill_(1)
+                finished_hyp = is_finished[i].nonzero().view(-1)
+
+                # Store finished hypotheses for this batch.
+                b = self.batch_offset[i]
+                for j in finished_hyp:
+                    self.hypotheses[b].append((topk_scores[i, j], predictions[i, j, :]))
+
+                # If the batch reached the end, save the best hypotheses
+                # in terms of length-penalized score.
+                if is_top_beam_finished[i]:
+                    best_hyp = sorted(
+                        self.hypotheses[b], key=lambda x: x[0], reverse=True
+                    )
+                    best_score, best_prediction = best_hyp[0]
+                    self.results["scores"][b].append(best_score)
+                    self.results["predictions"][b].append(best_prediction)
+
+            non_finished = is_top_beam_finished.eq(0).nonzero().view(-1)
+            if len(non_finished) == 0:
+                self.is_done = True
+
+            # Remove finished batches for the next step.
+            topk_log_probabilities = topk_log_probabilities.index_select(
+                0, non_finished
+            )
+            self.batch_offset = self.batch_offset.index_select(0, non_finished)
+            self.growing_beam = predictions.index_select(0, non_finished).view(
+                -1, self.growing_beam.size(-1)
+            )
+
+            surviving_beams_rows = surviving_beams_rows.index_select(0, non_finished)
+
+        return surviving_beams_rows
+
+    def forward(self, encoder_input_ids, **kwargs):
+        # keyword arguments come in 3 flavors: encoder-specific (prefixed by
+        # `encoder_`), decoder-specific (prefixed by `decoder_`) and those
+        # that apply to the model as whole.
+        # We let the specific kwargs override the common ones in case of conflict.
         kwargs_encoder = {
-            argument: value
+            argument[len("encoder_"):]: value
             for argument, value in kwargs.items()
-            if not argument.startswith("decoder_")
+            if argument.startswith("encoder_")
         }
         kwargs_decoder = {
             argument[len("decoder_"):]: value
             for argument, value in kwargs.items()
             if argument.startswith("decoder_")
         }
+        kwargs_common = {
+            argument: value
+            for argument, value in kwargs.items()
+            if not (argument.startswith("encoder_") or argument.startswith("decoder_"))
+        }
+        kwargs_decoder = dict(kwargs_common, **kwargs_decoder)
+        kwargs_encoder = dict(kwargs_common, **kwargs_encoder)
 
-        batch_size, _ = input_ids.size(0)
-
-        # Variables that keep track of the status of the search
-        hypotheses = [[] for _ in range(batch_size)]
-        batch_offset = torch.arange(batch_size, dtype=torch.long)
-        beam_offset = torch.arange(
-            0,
-            batch_size * self.beam_size,
-            step=self.beam_size,
-            dtype=torch.long,
-        )
-        growing_beam = torch.full(
-            (batch_size * self.beam_size, 1),
-            self.start_token_id,
-            dtype=torch.long,
-        )
-        topk_log_probabilities = torch.tensor(
-            [0.0] + [float("-inf")] * (self.beam_size - 1),
-            dtype=torch.float,
-        ).repeat(batch_size)
-
-        # Forward pass on the encoder
-        encoder_outputs = self.encoder(input_ids, kwargs_encoder)
+        # forward pass on the encoder
+        encoder_outputs = self.model.encoder.forward(encoder_input_ids, kwargs_encoder)
         kwargs_decoder["encoder_hidden_states"] = tile(
             encoder_outputs, self.beam_size, dim=0
         )
 
-        results = {}
-        results["predictions"] = [[] for _ in batch_size]
-        results["scores"] = [[] for _ in batch_size]
-
+        # grow the beam by generating sequences in an autoregressive way
+        self.growing_beam = torch.full(
+            (self.batch_size * self.beam_size, 1), self.start_token_id, dtype=torch.long
+        )
         for step in range(self.max_length):
-            decoder_input = growing_beam[:, -1]
-            outputs = self.decoder(decoder_input, kwargs_decoder)
+            decoder_input = self.growing_beam[:, -1]
+            outputs = self.model.decoder(decoder_input, kwargs_decoder)
             log_probabilities = torch.nn.functional.log_softmax(outputs[1])
-            vocab_size = log_probabilities.size(-1)
+            surviving_beams_rows = self.step(log_probabilities)
+            if self.is_done:
+                break
 
-            # The batch size changes as some beams finish so we define:
-            _B = log_probabilities.size(0) // self.beam_size
-
-            # Multiply each beam probability with the probability of the
-            # next token (conditioned on the words in the beam).
-            log_probabilities += topk_log_probabilities.view(-1, 1)
-
-            # if the beam has not attained the minimum required length we
-            # make the end token arbitrarily unlikely.
-            if step < self.min_length:
-                log_probabilities[self.end_token_id] = -1e20
-
-            # Remove repeating tri-grams
-            if(self.args.block_trigram):
-                if(step + 1 > 3):
-                    for i in range(_B * self.beam_size):
-                        tokens = [t for t in growing_beam[i]]
-                        trigrams = [(tokens[i-1], tokens[i], tokens[i+1]) for i in range(1, len(words) - 1)]
-                        last_trigram = tuple(trigrams[-1])
-                        if last_trigram in trigrams[:-1]:
-                            log_probabilities[i] = -1e20
-
-            # Find the `beam_size` (previous_beam + token) combinations with
-            # the highest score
-            topk_log_probabilities, topk_ids = log_probabilities.topk(
-                log_probabilities.view(_B, self.beam_size * vocab_size),
-                self.beam_size,
-                dim=1
-            )
-
-            # Apply the length penalty. The +1 accounts for the [EOS] token
-            # that will be added if the beam ends.
-            length_penalty = ((5.0 + (step + 1)) / 6.0) ** self.alpha
-            topk_scores = topk_log_probabilities / length_penalty
-
-            # Retrieve the corresponding respective beam and token id
-            # topk_token_ids[i] will be added to topk_beam_ids[i]
-            topk_beam_ids = topk_ids.div(vocab_size)
-            topk_token_ids = topk_ids.fmod(vocab_size)
-
-            # Retrieve the row index of the surviving beams in the original
-            # view of the log_probabilities tensor
-            surviving_beams_rows = (
-                topk_beam_ids + beam_offset[:_B].view(-1, 1)
-            ).view(-1)
-
-            # Append the last predictions
-            growing_beam = torch.cat(
-                [
-                    growing_beam.index_select(0, surviving_beams_rows),
-                    topk_token_ids.view(-1, 1),
-                ],
-                1,
-            )
-
-            # Check if any of the beam searches has ended during this
-            # growth step. Also if top beam (most probable) has ended
-            # for one element of the batch.
-            is_finished = topk_token_ids.eq(self.end_token_id)
-            if step + 1 == self.max_length:
-                is_finished.fill_(1)
-            is_top_beam_finished = is_finished[:, 0].eq(1)
-
-            # Save the finished searches
-            if is_finished.any():
-                predictions = growing_beam.view(-1, self.beam_size, growing_beam.size(1))
-                for i in range(is_finished.size(0)):
-                    if is_top_beam_finished[i]:
-                        is_finished[i].fill_(1)
-                    finished_hyp = is_finished[i].nonzero().view(-1)
-
-                    # Store finished hypotheses for this batch.
-                    b = batch_offset[i]
-                    for j in finished_hyp:
-                        hypotheses[b].append((topk_scores[i, j], predictions[i, j, :]))
-
-                    # If the batch reached the end, save the best hypotheses
-                    # in terms of length-penalized score.
-                    if is_top_beam_finished[i]:
-                        best_hyp = sorted(
-                            hypotheses[b], key=lambda x: x[0], reverse=True
-                        )
-                        best_score, best_prediction = best_hyp[0]
-                        results["scores"][b].append(best_score)
-                        results["predictions"][b].append(best_prediction)
-
-                non_finished = is_top_beam_finished.eq(0).nonzero().view(-1)
-                if len(non_finished) == 0:
-                    break
-
-                # Remove finished batches for the next step.
-                topk_log_probabilities = topk_log_probabilities.index_select(0, non_finished)
-                batch_offset = batch_offset.index_select(0, non_finished)
-                growing_beam = predictions.index_select(0, non_finished).view(
-                    -1, growing_beam.size(-1)
-                )
-
-            # Re-order the state for the next pass
-            surviving_beams_rows = surviving_beams_rows.index_select(0, non_finished)
             kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[
                 "encoder_hidden_states"
             ].index_select(0, surviving_beams_rows)
 
-        return results
+        return self.results
+
+    def remove_repeating_trigrams(self, log_probabilities, _B):
+        if(self._step + 1 > 3):
+            for i in range(_B * self.beam_size):
+                tokens = [t for t in self.growing_beam[i]]
+                trigrams = [(tokens[i-1], tokens[i], tokens[i+1]) for i in range(1, len(words) - 1)]
+                last_trigram = tuple(trigrams[-1])
+                if last_trigram in trigrams[:-1]:
+                    log_probabilities[i] = -1e20
+
+    def enforce_min_length(self):
+        if self._step < self.min_length:
+            self.log_probabilities[self.end_token_id] = -1e20
+
+    def enforce_max_length(self):
+        if self._step + 1 == self.max_length:
+            self.is_finished.fill_(1)
+
+    def length_penalty(self):
+        return ((5.0 + (self._step + 1)) / 6.0) ** self.alpha
 
 
 def tile(x, count, dim=0):
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 93f3c7e1f1..1081c8dd7b 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -632,6 +632,8 @@ class BertModel(BertPreTrainedModel):
         """
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
+        if encoder_attention_mask is None:
+            encoder_attention_mask = torch.ones_like(input_ids)
         if token_type_ids is None:
             token_type_ids = torch.zeros_like(input_ids)
 
@@ -660,12 +662,15 @@ class BertModel(BertPreTrainedModel):
         extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
         extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
 
-        # If a 2D encoder attention mask is provided for the cross-attention
+        # If a 2D ou 3D attention mask is provided for the cross-attention
         # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
-        if encoder_attention_mask is not None:
-            encoder_attention_mask = encoder_attention_mask[:, None, None, :]
-            encoder_attention_mask = encoder_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
-            encoder_attention_mask = (1.0 - encoder_attention_mask) * -10000.0
+        if encoder_attention_mask.dim() == 3:
+            encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
+        if encoder_attention_mask.dim() == 2:
+            encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
+
+        encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+        encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0
 
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
@@ -687,7 +692,7 @@ class BertModel(BertPreTrainedModel):
                                        attention_mask=extended_attention_mask,
                                        head_mask=head_mask,
                                        encoder_hidden_states=encoder_hidden_states,
-                                       encoder_attention_mask=encoder_attention_mask)
+                                       encoder_attention_mask=encoder_extended_attention_mask)
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
@@ -788,8 +793,10 @@ class BertForMaskedLM(BertPreTrainedModel):
             in ``[0, ..., config.vocab_size]``
 
     Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
-        **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+        **masked_lm_loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
             Masked language modeling loss.
+        **next_token_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Next token prediction loss.
         **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
             Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
         **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
@@ -854,13 +861,13 @@ class BertForMaskedLM(BertPreTrainedModel):
 
         if lm_labels is not None:
             # we are doing next-token prediction; shift prediction scores and input ids by one
-            prediction_scores = prediction_scores[:, :-1, :]
-            lm_labels = lm_labels[:, 1:]
+            prediction_scores = prediction_scores[:, :-1, :].contiguous()
+            lm_labels = lm_labels[:, 1:].contiguous()
             loss_fct = CrossEntropyLoss(ignore_index=-1)
-            seq2seq_loss = loss_fct(prediction_scores.reshape(-1, self.config.vocab_size), lm_labels.reshape(-1))
-            outputs = (seq2seq_loss,) + outputs
+            next_token_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), lm_labels.view(-1))
+            outputs = (next_token_loss,) + outputs
 
-        return outputs  # (mlm_or_seq2seq_loss), prediction_scores, (hidden_states), (attentions)
+        return outputs  # (masked_lm_loss), (next_token_loss), prediction_scores, (hidden_states), (attentions)
 
 
 @add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """,
diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 2767dd2cd1..22898db9a1 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 class PreTrainedSeq2seq(nn.Module):
     r"""
-        :class:`~transformers.Seq2seq` is a generic model class that will be
+        :class:`~transformers.PreTrainedSeq2seq` is a generic model class that will be
         instantiated as a Seq2seq model with one of the base model classes of
         the library as encoder and (optionally) as decoder when created with
         the `AutoModel.from_pretrained(pretrained_model_name_or_path)` class
@@ -49,8 +49,7 @@ class PreTrainedSeq2seq(nn.Module):
         *model_args,
         **kwargs
     ):
-        r""" Instantiates an encoder and a decoder from one or two base classes
-        of the library from pre-trained model checkpoints.
+        r""" Instantiates an encoder and a decoder from one or two base classes of the library from pre-trained model checkpoints.
 
 
         The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
@@ -111,35 +110,44 @@ class PreTrainedSeq2seq(nn.Module):
             model = PreTrainedSeq2seq.from_pretained('bert-base-uncased', 'bert-base-uncased') # initialize Bert2Bert
         """
 
-        # Separate the encoder- and decoder- specific kwargs. A kwarg is
-        # decoder-specific it the key starts with `decoder_`
+        # keyword arguments come in 3 flavors: encoder-specific (prefixed by
+        # `encoder_`), decoder-specific (prefixed by `decoder_`) and those
+        # that apply to the model as a whole.
+        # We let the specific kwargs override the common ones in case of conflict.
         kwargs_encoder = {
-            argument: value
+            argument[len("encoder_"):]: value
             for argument, value in kwargs.items()
-            if not argument.startswith("decoder_")
+            if argument.startswith("encoder_")
         }
         kwargs_decoder = {
-            argument[len("decoder_") :]: value
+            argument[len("decoder_"):]: value
             for argument, value in kwargs.items()
             if argument.startswith("decoder_")
         }
+        kwargs_common = {
+            argument: value
+            for argument, value in kwargs.items()
+            if not (argument.startswith("encoder_") or argument.startswith("decoder_"))
+        }
+        kwargs_decoder = dict(kwargs_common, **kwargs_decoder)
+        kwargs_encoder = dict(kwargs_common, **kwargs_encoder)
 
         # Load and initialize the encoder and decoder
         # The distinction between encoder and decoder at the model level is made
         # by the value of the flag `is_decoder` that we need to set correctly.
-        encoder = kwargs_encoder.pop("encoder_model", None)
+        encoder = kwargs_encoder.pop("model", None)
         if encoder is None:
-            kwargs_encoder["is_decoder"] = False
             encoder = AutoModel.from_pretrained(
                 encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder
             )
+        encoder.config.is_decoder = False
 
         decoder = kwargs_decoder.pop("model", None)
         if decoder is None:
-            kwargs_decoder["is_decoder"] = True
             decoder = AutoModelWithLMHead.from_pretrained(
                 decoder_pretrained_model_name_or_path, **kwargs_decoder
             )
+        decoder.config.is_decoder = True
 
         model = cls(encoder, decoder)
 
@@ -169,37 +177,60 @@ class PreTrainedSeq2seq(nn.Module):
             decoder_input_ids: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``
                 Indices of decoder input sequence tokens in the vocabulary.
         """
-        # Separate the encoder- and decoder- specific kwargs. A kwarg is
-        # decoder-specific it the key starts with `decoder_`
+        # keyword arguments come in 3 flavors: encoder-specific (prefixed by
+        # `encoder_`), decoder-specific (prefixed by `decoder_`) and those
+        # that apply to the model as whole.
+        # We let the specific kwargs override the common ones in case of conflict.
         kwargs_encoder = {
-            argument: value
+            argument[len("encoder_"):]: value
             for argument, value in kwargs.items()
-            if not argument.startswith("decoder_")
+            if argument.startswith("encoder_")
         }
         kwargs_decoder = {
-            argument[len("decoder_") :]: value
+            argument[len("decoder_"):]: value
             for argument, value in kwargs.items()
             if argument.startswith("decoder_")
         }
+        kwargs_common = {
+            argument: value
+            for argument, value in kwargs.items()
+            if not (argument.startswith("encoder_") or argument.startswith("decoder_"))
+        }
+        kwargs_decoder = dict(kwargs_common, **kwargs_decoder)
+        kwargs_encoder = dict(kwargs_common, **kwargs_encoder)
 
         # Encode if needed (training, first prediction pass)
-        encoder_hidden_states = kwargs_encoder.pop("encoder_hidden_states", None)
+        encoder_hidden_states = kwargs_encoder.pop("hidden_states", None)
         if encoder_hidden_states is None:
             encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder)
-            encoder_hidden_states = encoder_outputs[0][
-                -1
-            ]  # output of the encoder *stack*
+            encoder_hidden_states = encoder_outputs[0]  # output the last layer hidden state
         else:
             encoder_outputs = ()
 
         # Decode
-        kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states[None, :, :]
+        kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states
+        kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None)
         decoder_outputs = self.decoder(decoder_input_ids, **kwargs_decoder)
 
         return decoder_outputs + encoder_outputs
 
 
 class Model2Model(PreTrainedSeq2seq):
+    r"""
+        :class:`~transformers.Model2Model` instantiates a Seq2Seq2 model
+        where both of the encoder and decoder are of the same family. If the
+        name of or that path to a pretrained model is specified the encoder and
+        the decoder will be initialized with the pretrained weight (the
+        cross-attention will be intialized randomly if its weights are not
+        present).
+
+        It is possible to override this behavior and initialize, say, the decoder randomly
+        by creating it beforehand as follows
+
+            config = BertConfig.from_pretrained()
+            decoder = BertForMaskedLM(config)
+            model = Model2Model.from_pretrained('bert-base-uncased', decoder_model=decoder)
+    """
     def __init__(self, *args, **kwargs):
         super(Model2Model, self).__init__(*args, **kwargs)
         self.tie_weights()
@@ -235,14 +266,10 @@ class Model2Model(PreTrainedSeq2seq):
         model = super(Model2Model, cls).from_pretrained(
             encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
+            *args,
             **kwargs
         )
 
-        # Some architectures require for the decoder to be initialized randomly
-        # before fine-tuning.
-        if kwargs.get("decoder_initialize_randomly", False):
-            model.decoder.init_weights()
-
         return model
 
 

From 098a89f312311a730275a79af7cf5c527d35fdd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Tue, 29 Oct 2019 20:08:03 +0100
Subject: [PATCH 100/269] update docstrings; rename lm_labels to more explicit
 ltr_lm_labels

---
 examples/run_summarization_finetuning.py |  8 ++--
 transformers/modeling_bert.py            | 51 +++++++++++++-----------
 2 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py
index 1888f56caf..2dc8c660ce 100644
--- a/examples/run_summarization_finetuning.py
+++ b/examples/run_summarization_finetuning.py
@@ -26,7 +26,7 @@ import numpy as np
 from tqdm import tqdm, trange
 import torch
 from torch.optim import Adam
-from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 
 from transformers import (
     AutoTokenizer,
@@ -283,14 +283,14 @@ def evaluate(args, model, tokenizer, prefix=""):
     model.eval()
 
     for batch in tqdm(eval_dataloader, desc="Evaluating"):
-        source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch
+        source, target, encoder_token_type_ids, encoder_mask, decoder_mask, ltr_lm_labels = batch
 
         source = source.to(args.device)
         target = target.to(args.device)
         encoder_token_type_ids = encoder_token_type_ids.to(args.device)
         encoder_mask = encoder_mask.to(args.device)
         decoder_mask = decoder_mask.to(args.device)
-        lm_labels = lm_labels.to(args.device)
+        ltr_lm_labels = ltr_lm_labels.to(args.device)
 
         with torch.no_grad():
             outputs = model(
@@ -299,7 +299,7 @@ def evaluate(args, model, tokenizer, prefix=""):
                 encoder_token_type_ids=encoder_token_type_ids,
                 encoder_attention_mask=encoder_mask,
                 decoder_attention_mask=decoder_mask,
-                decoder_lm_labels=lm_labels,
+                decoder_ltr_lm_labels=ltr_lm_labels,
             )
             lm_loss = outputs[0]
             eval_loss += lm_loss.mean().item()
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 1081c8dd7b..3fec69a814 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -548,6 +548,14 @@ BERT_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **encoder_hidden_states**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``:
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model
+            is configured as a decoder.
+        **encoder_attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask
+            is used in the cross-attention if the model is configured as a decoder.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
 """
 
 @add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.",
@@ -609,26 +617,18 @@ class BertModel(BertPreTrainedModel):
                 head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
         """ Forward pass on the Model.
 
-        The values of the attention matrix (shape [batch_size, seq_length])
-        should be 1.0 for the position we want to attend to and 0. for the ones
-        we do not want to attend to.
-
         The model can behave as an encoder (with only self-attention) as well
         as a decoder, in which case a layer of cross-attention is added between
-        ever self-attention layer, following the architecture described in [1].
+        the self-attention layers, following the architecture described in `Attention is all you need`_ by Ashish Vaswani,
+        Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
 
-        To behave like as a decoder the model needs to be initialized with the
-        `is_decoder` argument of the config set to `True`. An
+        To behave as an decoder the model needs to be initialized with the
+        `is_decoder` argument of the configuration set to `True`; an
         `encoder_hidden_states` is expected as an input to the forward pass.
-        When a decoder, there are two kinds of attention masks to specify:
 
-        (1) Self-attention masks that need to be causal (only attends to
-        previous tokens);
-        (2) A cross-attention mask that prevents the module
-        from attending to the encoder's padding tokens.
+        .. _`Attention is all you need`:
+            https://arxiv.org/abs/1706.03762
 
-        [1] Vaswani, Ashish, et al. "Attention is all you need." Advances in
-        neural information processing systems. 2017.
         """
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
@@ -791,11 +791,16 @@ class BertForMaskedLM(BertPreTrainedModel):
             Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
             Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
             in ``[0, ..., config.vocab_size]``
+        **ltr_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the left-to-right language modeling loss (next word prediction).
+            Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
+            Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
+            in ``[0, ..., config.vocab_size]``
 
     Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
         **masked_lm_loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
             Masked language modeling loss.
-        **next_token_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+        **ltr_lm_loss**: (`optional`, returned when ``ltr_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
             Next token prediction loss.
         **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
             Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
@@ -833,7 +838,7 @@ class BertForMaskedLM(BertPreTrainedModel):
                                    self.bert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, lm_labels=None, ):
+                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, ltr_lm_labels=None, ):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
@@ -852,22 +857,22 @@ class BertForMaskedLM(BertPreTrainedModel):
         # 1. If a tensor that contains the indices of masked labels is provided,
         #    the cross-entropy is the MLM cross-entropy that measures the likelihood
         #    of predictions for masked words.
-        # 2. If `lm_label` is provided we are in a causal scenario where we
-        #    try to predict the next word for each input in the encoder.
+        # 2. If `ltr_lm_labels` is provided we are in a causal scenario where we
+        #    try to predict the next token for each input in the decoder.
         if masked_lm_labels is not None:
             loss_fct = CrossEntropyLoss(ignore_index=-1)  # -1 index = padding token
             masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
             outputs = (masked_lm_loss,) + outputs
 
-        if lm_labels is not None:
+        if ltr_lm_labels is not None:
             # we are doing next-token prediction; shift prediction scores and input ids by one
             prediction_scores = prediction_scores[:, :-1, :].contiguous()
-            lm_labels = lm_labels[:, 1:].contiguous()
+            ltr_lm_labels = ltr_lm_labels[:, 1:].contiguous()
             loss_fct = CrossEntropyLoss(ignore_index=-1)
-            next_token_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), lm_labels.view(-1))
-            outputs = (next_token_loss,) + outputs
+            ltr_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), ltr_lm_labels.view(-1))
+            outputs = (ltr_lm_loss,) + outputs
 
-        return outputs  # (masked_lm_loss), (next_token_loss), prediction_scores, (hidden_states), (attentions)
+        return outputs  # (masked_lm_loss), (ltr_lm_loss), prediction_scores, (hidden_states), (attentions)
 
 
 @add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """,

From 842f3bf049d4a728cb4bff543e8bbd74020af230 Mon Sep 17 00:00:00 2001
From: Timothy Liu <tlkh.xms@gmail.com>
Date: Wed, 30 Oct 2019 01:32:15 +0000
Subject: [PATCH 101/269] Fixed training for TF XLM

---
 transformers/modeling_tf_xlm.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py
index 84de1517ee..9ac5d28e1f 100644
--- a/transformers/modeling_tf_xlm.py
+++ b/transformers/modeling_tf_xlm.py
@@ -84,7 +84,8 @@ def get_masks(slen, lengths, causal, padding_mask=None, dtype=tf.float32):
         attn_mask = mask
 
     # sanity check
-    assert shape_list(mask) == [bs, slen]
+    # assert shape_list(mask) == [bs, slen]
+    tf.debugging.assert_equal(shape_list(mask), [bs, slen])
     assert causal is False or shape_list(attn_mask) == [bs, slen, slen]
 
     mask = tf.cast(mask, dtype=dtype)
@@ -318,7 +319,8 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
 
         # check inputs
         bs, slen = shape_list(input_ids)
-        assert shape_list(lengths)[0] == bs
+        # assert shape_list(lengths)[0] == bs
+        tf.debugging.assert_equal(shape_list(lengths)[0], bs)
         # assert lengths.max().item() <= slen
         # input_ids = input_ids.transpose(0, 1)  # batch size as dimension 0
         # assert (src_enc is None) == (src_len is None)
@@ -335,12 +337,14 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
         if position_ids is None:
             position_ids = tf.expand_dims(tf.range(slen), axis=0)
         else:
-            assert shape_list(position_ids) == [bs, slen]  # (slen, bs)
+            # assert shape_list(position_ids) == [bs, slen]  # (slen, bs)
+            tf.debugging.assert_equal(shape_list(position_ids), [bs, slen])
             # position_ids = position_ids.transpose(0, 1)
 
         # langs
         if langs is not None:
-            assert shape_list(langs) == [bs, slen]  # (slen, bs)
+            # assert shape_list(langs) == [bs, slen]  # (slen, bs)
+            tf.debugging.assert_equal(shape_list(langs), [bs, slen])
             # langs = langs.transpose(0, 1)
 
         # Prepare head mask if needed

From 9c1bdb5b61303bbdfbc3b9759f5c5fa847cb377d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 10:43:13 +0100
Subject: [PATCH 102/269] revert renaming of lm_labels to ltr_lm_labels

---
 examples/run_summarization_finetuning.py |  6 +++---
 transformers/modeling_bert.py            | 14 +++++++-------
 transformers/modeling_seq2seq.py         | 22 +++++++++++++---------
 3 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py
index 2dc8c660ce..3d194950c7 100644
--- a/examples/run_summarization_finetuning.py
+++ b/examples/run_summarization_finetuning.py
@@ -283,14 +283,14 @@ def evaluate(args, model, tokenizer, prefix=""):
     model.eval()
 
     for batch in tqdm(eval_dataloader, desc="Evaluating"):
-        source, target, encoder_token_type_ids, encoder_mask, decoder_mask, ltr_lm_labels = batch
+        source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch
 
         source = source.to(args.device)
         target = target.to(args.device)
         encoder_token_type_ids = encoder_token_type_ids.to(args.device)
         encoder_mask = encoder_mask.to(args.device)
         decoder_mask = decoder_mask.to(args.device)
-        ltr_lm_labels = ltr_lm_labels.to(args.device)
+        lm_labels = lm_labels.to(args.device)
 
         with torch.no_grad():
             outputs = model(
@@ -299,7 +299,7 @@ def evaluate(args, model, tokenizer, prefix=""):
                 encoder_token_type_ids=encoder_token_type_ids,
                 encoder_attention_mask=encoder_mask,
                 decoder_attention_mask=decoder_mask,
-                decoder_ltr_lm_labels=ltr_lm_labels,
+                decoder_lm_labels=lm_labels,
             )
             lm_loss = outputs[0]
             eval_loss += lm_loss.mean().item()
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 3fec69a814..11fcdde685 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -791,7 +791,7 @@ class BertForMaskedLM(BertPreTrainedModel):
             Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
             Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
             in ``[0, ..., config.vocab_size]``
-        **ltr_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+        **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
             Labels for computing the left-to-right language modeling loss (next word prediction).
             Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
             Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
@@ -800,7 +800,7 @@ class BertForMaskedLM(BertPreTrainedModel):
     Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
         **masked_lm_loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
             Masked language modeling loss.
-        **ltr_lm_loss**: (`optional`, returned when ``ltr_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+        **ltr_lm_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
             Next token prediction loss.
         **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
             Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
@@ -838,7 +838,7 @@ class BertForMaskedLM(BertPreTrainedModel):
                                    self.bert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, ltr_lm_labels=None, ):
+                masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, lm_labels=None, ):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
@@ -857,19 +857,19 @@ class BertForMaskedLM(BertPreTrainedModel):
         # 1. If a tensor that contains the indices of masked labels is provided,
         #    the cross-entropy is the MLM cross-entropy that measures the likelihood
         #    of predictions for masked words.
-        # 2. If `ltr_lm_labels` is provided we are in a causal scenario where we
+        # 2. If `lm_labels` is provided we are in a causal scenario where we
         #    try to predict the next token for each input in the decoder.
         if masked_lm_labels is not None:
             loss_fct = CrossEntropyLoss(ignore_index=-1)  # -1 index = padding token
             masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
             outputs = (masked_lm_loss,) + outputs
 
-        if ltr_lm_labels is not None:
+        if lm_labels is not None:
             # we are doing next-token prediction; shift prediction scores and input ids by one
             prediction_scores = prediction_scores[:, :-1, :].contiguous()
-            ltr_lm_labels = ltr_lm_labels[:, 1:].contiguous()
+            lm_labels = lm_labels[:, 1:].contiguous()
             loss_fct = CrossEntropyLoss(ignore_index=-1)
-            ltr_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), ltr_lm_labels.view(-1))
+            ltr_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), lm_labels.view(-1))
             outputs = (ltr_lm_loss,) + outputs
 
         return outputs  # (masked_lm_loss), (ltr_lm_loss), prediction_scores, (hidden_states), (attentions)
diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_seq2seq.py
index 22898db9a1..ba8c546a30 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_seq2seq.py
@@ -30,10 +30,10 @@ logger = logging.getLogger(__name__)
 class PreTrainedSeq2seq(nn.Module):
     r"""
         :class:`~transformers.PreTrainedSeq2seq` is a generic model class that will be
-        instantiated as a Seq2seq model with one of the base model classes of
-        the library as encoder and (optionally) as decoder when created with
-        the `AutoModel.from_pretrained(pretrained_model_name_or_path)` class
-        method.
+        instantiated as a transformer architecture with one of the base model
+        classes of the library as encoder and (optionally) another one as
+        decoder when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)`
+        class method.
     """
 
     def __init__(self, encoder, decoder):
@@ -59,13 +59,13 @@ class PreTrainedSeq2seq(nn.Module):
             encoder_pretrained_model_name_or_path: information necessary to initiate the encoder. Either:
 
                 - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
-                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
+                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/encoder``.
                 - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
 
             decoder_pretrained_model_name_or_path: information necessary to initiate the decoder. Either:
 
                 - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
-                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
+                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/decoder``.
                 - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
 
             model_args: (`optional`) Sequence of positional arguments:
@@ -103,7 +103,7 @@ class PreTrainedSeq2seq(nn.Module):
                 - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
                 - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
 
-                You can specify different kwargs for the decoder by prefixing the key with `decoder_` (e.g. ``decoder_output_attention=True``).
+                You can specify kwargs sepcific for the encoder and decoder by prefixing the key with `encoder_` and `decoder_` respectively. (e.g. ``decoder_output_attention=True``). The remaining kwargs will be passed to both encoders and decoders.
 
         Examples::
 
@@ -154,8 +154,11 @@ class PreTrainedSeq2seq(nn.Module):
         return model
 
     def save_pretrained(self, save_directory):
-        """ Save a Seq2Seq model and its configuration file in a format
-        such that it can be loaded using `:func:`~transformers.PreTrainedSeq2seq.from_pretrained` """
+        """ Save a Seq2Seq model and its configuration file in a format such
+        that it can be loaded using `:func:`~transformers.PreTrainedSeq2seq.from_pretrained`
+
+        We save the encoder' and decoder's parameters in two separate directories.
+        """
         self.encoder.save_pretrained(os.path.join(save_directory, "encoder"))
         self.decoder.save_pretrained(os.path.join(save_directory, "decoder"))
 
@@ -176,6 +179,7 @@ class PreTrainedSeq2seq(nn.Module):
                 Indices of encoder input sequence tokens in the vocabulary.
             decoder_input_ids: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``
                 Indices of decoder input sequence tokens in the vocabulary.
+            kwargs: (`optional`) Remaining dictionary of keyword arguments.
         """
         # keyword arguments come in 3 flavors: encoder-specific (prefixed by
         # `encoder_`), decoder-specific (prefixed by `decoder_`) and those

From 3b0d2fa30eb9756c888b4ed36213350d4b6e70e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 10:54:46 +0100
Subject: [PATCH 103/269] rename seq2seq to encoder_decoder

---
 examples/README.md                             |  6 ++----
 examples/run_summarization_finetuning.py       |  4 ++--
 transformers/__init__.py                       |  2 +-
 ..._seq2seq.py => modeling_encoder_decoder.py} | 18 +++++++++---------
 4 files changed, 14 insertions(+), 16 deletions(-)
 rename transformers/{modeling_seq2seq.py => modeling_encoder_decoder.py} (96%)

diff --git a/examples/README.md b/examples/README.md
index bec6d57171..6d27a0c560 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -10,7 +10,7 @@ similar API between the different models.
 | [GLUE](#glue) | Examples running BERT/XLM/XLNet/RoBERTa on the 9 GLUE tasks. Examples feature distributed training as well as half-precision.                              |
 | [SQuAD](#squad) | Using BERT for question answering, examples with distributed training.                                                                                  |
 | [Multiple Choice](#multiple choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. 
-| [Seq2seq Model fine-tuning](#seq2seq-model-fine-tuning) | Fine-tuning the library models for seq2seq tasks on the CNN/Daily Mail dataset. |
+| [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. |
 
 ## Language model fine-tuning
 
@@ -391,7 +391,7 @@ exact_match = 86.91
 This fine-tuned model is available as a checkpoint under the reference
 `bert-large-uncased-whole-word-masking-finetuned-squad`.
 
-## Seq2seq model fine-tuning
+## Abstractive summarization
 
 Based on the script
 [`run_summarization_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_summarization_finetuning.py).
@@ -408,8 +408,6 @@ note that the finetuning script **will not work** if you do not download both
 datasets. We will refer as `$DATA_PATH` the path to where you uncompressed both
 archive.
 
-## Bert2Bert and abstractive summarization
-
 ```bash
 export DATA_PATH=/path/to/dataset/
 
diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py
index 3d194950c7..448505c727 100644
--- a/examples/run_summarization_finetuning.py
+++ b/examples/run_summarization_finetuning.py
@@ -32,7 +32,7 @@ from transformers import (
     AutoTokenizer,
     BertForMaskedLM,
     BertConfig,
-    PreTrainedSeq2seq,
+    PreTrainedEncoderDecoder,
     Model2Model,
 )
 
@@ -475,7 +475,7 @@ def main():
         for checkpoint in checkpoints:
             encoder_checkpoint = os.path.join(checkpoint, "encoder")
             decoder_checkpoint = os.path.join(checkpoint, "decoder")
-            model = PreTrainedSeq2seq.from_pretrained(
+            model = PreTrainedEncoderDecoder.from_pretrained(
                 encoder_checkpoint, decoder_checkpoint
             )
             model.to(args.device)
diff --git a/transformers/__init__.py b/transformers/__init__.py
index 2206a0302e..844aa22295 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -87,7 +87,7 @@ if is_torch_available():
     from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel,
                                 DistilBertForSequenceClassification, DistilBertForQuestionAnswering,
                                 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
-    from .modeling_seq2seq import PreTrainedSeq2seq, Model2Model
+    from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
     # Optimization
     from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule,
diff --git a/transformers/modeling_seq2seq.py b/transformers/modeling_encoder_decoder.py
similarity index 96%
rename from transformers/modeling_seq2seq.py
rename to transformers/modeling_encoder_decoder.py
index ba8c546a30..162e2f8b3b 100644
--- a/transformers/modeling_seq2seq.py
+++ b/transformers/modeling_encoder_decoder.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Auto Model class. """
+""" Classes to support Encoder-Decoder architectures """
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
@@ -27,9 +27,9 @@ from .modeling_auto import AutoModel, AutoModelWithLMHead
 logger = logging.getLogger(__name__)
 
 
-class PreTrainedSeq2seq(nn.Module):
+class PreTrainedEncoderDecoder(nn.Module):
     r"""
-        :class:`~transformers.PreTrainedSeq2seq` is a generic model class that will be
+        :class:`~transformers.PreTrainedEncoderDecoder` is a generic model class that will be
         instantiated as a transformer architecture with one of the base model
         classes of the library as encoder and (optionally) another one as
         decoder when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)`
@@ -37,7 +37,7 @@ class PreTrainedSeq2seq(nn.Module):
     """
 
     def __init__(self, encoder, decoder):
-        super(PreTrainedSeq2seq, self).__init__()
+        super(PreTrainedEncoderDecoder, self).__init__()
         self.encoder = encoder
         self.decoder = decoder
 
@@ -107,7 +107,7 @@ class PreTrainedSeq2seq(nn.Module):
 
         Examples::
 
-            model = PreTrainedSeq2seq.from_pretained('bert-base-uncased', 'bert-base-uncased') # initialize Bert2Bert
+            model = PreTrainedEncoderDecoder.from_pretained('bert-base-uncased', 'bert-base-uncased') # initialize Bert2Bert
         """
 
         # keyword arguments come in 3 flavors: encoder-specific (prefixed by
@@ -155,7 +155,7 @@ class PreTrainedSeq2seq(nn.Module):
 
     def save_pretrained(self, save_directory):
         """ Save a Seq2Seq model and its configuration file in a format such
-        that it can be loaded using `:func:`~transformers.PreTrainedSeq2seq.from_pretrained`
+        that it can be loaded using `:func:`~transformers.PreTrainedEncoderDecoder.from_pretrained`
 
         We save the encoder' and decoder's parameters in two separate directories.
         """
@@ -219,7 +219,7 @@ class PreTrainedSeq2seq(nn.Module):
         return decoder_outputs + encoder_outputs
 
 
-class Model2Model(PreTrainedSeq2seq):
+class Model2Model(PreTrainedEncoderDecoder):
     r"""
         :class:`~transformers.Model2Model` instantiates a Seq2Seq2 model
         where both of the encoder and decoder are of the same family. If the
@@ -277,14 +277,14 @@ class Model2Model(PreTrainedSeq2seq):
         return model
 
 
-class Model2LSTM(PreTrainedSeq2seq):
+class Model2LSTM(PreTrainedEncoderDecoder):
     @classmethod
     def from_pretrained(cls, *args, **kwargs):
         if kwargs.get("decoder_model", None) is None:
             # We will create a randomly initilized LSTM model as decoder
             if "decoder_config" not in kwargs:
                 raise ValueError(
-                    "To load an LSTM in Seq2seq model, please supply either: "
+                    "To load an LSTM in Encoder-Decoder model, please supply either: "
                     "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or"
                     "    - a dictionary of configuration parameters that will be used to initialize a"
                     "      torch.nn.LSTM model as `decoder_config` keyword argument. "

From da10de8466c001dceca328dac12751abb71c65eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 11:19:58 +0100
Subject: [PATCH 104/269] fix bug with padding mask + add corresponding test

---
 examples/utils_summarization.py      | 6 +++---
 examples/utils_summarization_test.py | 7 +++++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py
index cd8bc4bc2b..2a8f81cd36 100644
--- a/examples/utils_summarization.py
+++ b/examples/utils_summarization.py
@@ -127,9 +127,9 @@ def build_lm_labels(sequence, pad_token):
 def build_mask(sequence, pad_token):
     """ Builds the mask. The attention mechanism will only attend to positions
     with value 1. """
-    mask = sequence.clone()
-    mask[mask != pad_token] = 1
-    mask[mask == pad_token] = 0
+    mask = torch.ones_like(sequence)
+    idx_pad_tokens = (sequence == pad_token)
+    mask[idx_pad_tokens] = 0
     return mask
 
 
diff --git a/examples/utils_summarization_test.py b/examples/utils_summarization_test.py
index 7a02f8fa1f..7604bd185d 100644
--- a/examples/utils_summarization_test.py
+++ b/examples/utils_summarization_test.py
@@ -116,6 +116,13 @@ class SummarizationDataProcessingTest(unittest.TestCase):
             build_mask(sequence, 23).numpy(), expected.numpy()
         )
 
+    def test_build_mask_with_padding_equal_to_one(self):
+        sequence = torch.tensor([8, 2, 3, 4, 1, 1, 1])
+        expected = torch.tensor([1, 1, 1, 1, 0, 0, 0])
+        np.testing.assert_array_equal(
+            build_mask(sequence, 1).numpy(), expected.numpy()
+        )
+
     def test_compute_token_type_ids(self):
         separator = 101
         batch = torch.tensor(

From 070507df1ffd7609f4691089f0bbc7ac27df66fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 11:24:12 +0100
Subject: [PATCH 105/269] format utils for summarization

---
 examples/utils_summarization.py      | 2 +-
 examples/utils_summarization_test.py | 8 ++------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py
index 2a8f81cd36..327ca8cc3e 100644
--- a/examples/utils_summarization.py
+++ b/examples/utils_summarization.py
@@ -128,7 +128,7 @@ def build_mask(sequence, pad_token):
     """ Builds the mask. The attention mechanism will only attend to positions
     with value 1. """
     mask = torch.ones_like(sequence)
-    idx_pad_tokens = (sequence == pad_token)
+    idx_pad_tokens = sequence == pad_token
     mask[idx_pad_tokens] = 0
     return mask
 
diff --git a/examples/utils_summarization_test.py b/examples/utils_summarization_test.py
index 7604bd185d..1d56ff0803 100644
--- a/examples/utils_summarization_test.py
+++ b/examples/utils_summarization_test.py
@@ -105,9 +105,7 @@ class SummarizationDataProcessingTest(unittest.TestCase):
     def test_build_mask_no_padding(self):
         sequence = torch.tensor([1, 2, 3, 4])
         expected = torch.tensor([1, 1, 1, 1])
-        np.testing.assert_array_equal(
-            build_mask(sequence, 0).numpy(), expected.numpy()
-        )
+        np.testing.assert_array_equal(build_mask(sequence, 0).numpy(), expected.numpy())
 
     def test_build_mask(self):
         sequence = torch.tensor([1, 2, 3, 4, 23, 23, 23])
@@ -119,9 +117,7 @@ class SummarizationDataProcessingTest(unittest.TestCase):
     def test_build_mask_with_padding_equal_to_one(self):
         sequence = torch.tensor([8, 2, 3, 4, 1, 1, 1])
         expected = torch.tensor([1, 1, 1, 1, 0, 0, 0])
-        np.testing.assert_array_equal(
-            build_mask(sequence, 1).numpy(), expected.numpy()
-        )
+        np.testing.assert_array_equal(build_mask(sequence, 1).numpy(), expected.numpy())
 
     def test_compute_token_type_ids(self):
         separator = 101

From 7f4226f9e63639d23397ad89e2591b5d4fc35afc Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Wed, 30 Oct 2019 11:31:56 +0100
Subject: [PATCH 106/269] adding templates

---
 .../adding_a_new_example_script/README.md     |   5 +
 .../adding_a_new_example_script/run_xxx.py    | 553 ++++++++++
 .../adding_a_new_example_script/utils_xxx.py  | 995 ++++++++++++++++++
 templates/adding_a_new_model/README.md        |  62 ++
 .../adding_a_new_model/configuration_xxx.py   | 130 +++
 ...t_xxx_original_tf_checkpoint_to_pytorch.py |  65 ++
 .../adding_a_new_model/modeling_tf_xxx.py     | 500 +++++++++
 templates/adding_a_new_model/modeling_xxx.py  | 644 ++++++++++++
 .../tests/modeling_tf_xxx_test.py             | 256 +++++
 .../tests/modeling_xxx_test.py                | 255 +++++
 .../tests/tokenization_xxx_test.py            |  57 +
 .../adding_a_new_model/tokenization_xxx.py    | 218 ++++
 12 files changed, 3740 insertions(+)
 create mode 100644 templates/adding_a_new_example_script/README.md
 create mode 100644 templates/adding_a_new_example_script/run_xxx.py
 create mode 100644 templates/adding_a_new_example_script/utils_xxx.py
 create mode 100644 templates/adding_a_new_model/README.md
 create mode 100644 templates/adding_a_new_model/configuration_xxx.py
 create mode 100755 templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py
 create mode 100644 templates/adding_a_new_model/modeling_tf_xxx.py
 create mode 100644 templates/adding_a_new_model/modeling_xxx.py
 create mode 100644 templates/adding_a_new_model/tests/modeling_tf_xxx_test.py
 create mode 100644 templates/adding_a_new_model/tests/modeling_xxx_test.py
 create mode 100644 templates/adding_a_new_model/tests/tokenization_xxx_test.py
 create mode 100644 templates/adding_a_new_model/tokenization_xxx.py

diff --git a/templates/adding_a_new_example_script/README.md b/templates/adding_a_new_example_script/README.md
new file mode 100644
index 0000000000..2afca08bf8
--- /dev/null
+++ b/templates/adding_a_new_example_script/README.md
@@ -0,0 +1,5 @@
+# How to add a new example script in 🤗Transformers
+
+This folder provide a template for adding a new example script implementing a training or inference task with the models in the  🤗Transformers library.
+
+Currently only examples for PyTorch are provided which are adaptations of the library's SQuAD examples which implement single-GPU and distributed training with gradient accumulation and mixed-precision (using NVIDIA's apex library) to cover a reasonable range of use cases.
diff --git a/templates/adding_a_new_example_script/run_xxx.py b/templates/adding_a_new_example_script/run_xxx.py
new file mode 100644
index 0000000000..e348d9b5ea
--- /dev/null
+++ b/templates/adding_a_new_example_script/run_xxx.py
@@ -0,0 +1,553 @@
+# coding=utf-8
+# Copyright 2018 XXX.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for task XXX."""
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import logging
+import os
+import random
+import glob
+
+import numpy as np
+import torch
+from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
+                              TensorDataset)
+from torch.utils.data.distributed import DistributedSampler
+
+try:
+    from torch.utils.tensorboard import SummaryWriter
+except:
+    from tensorboardX import SummaryWriter
+
+from tqdm import tqdm, trange
+
+from transformers import (WEIGHTS_NAME, BertConfig,
+                                  BertForQuestionAnswering, BertTokenizer,
+                                  XLMConfig, XLMForQuestionAnswering,
+                                  XLMTokenizer, XLNetConfig,
+                                  XLNetForQuestionAnswering,
+                                  XLNetTokenizer,
+                                  DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
+
+from transformers import AdamW, WarmupLinearSchedule
+
+from utils_squad import (read_squad_examples, convert_examples_to_features,
+                         RawResult, write_predictions,
+                         RawResultExtended, write_predictions_extended)
+
+# The follwing import is the official SQuAD evaluation script (2.0).
+# You can remove it from the dependencies if you are using this script outside of the library
+# We've added it here for automated tests (see examples/test_examples.py file)
+from utils_squad_evaluate import EVAL_OPTS, main as evaluate_on_squad
+
+logger = logging.getLogger(__name__)
+
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) \
+                  for conf in (BertConfig, XLNetConfig, XLMConfig)), ())
+
+MODEL_CLASSES = {
+    'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer),
+    'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
+    'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
+    'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
+}
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+def to_list(tensor):
+    return tensor.detach().cpu().tolist()
+
+def train(args, train_dataset, model, tokenizer):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter()
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
+        ]
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
+                                                          output_device=args.local_rank,
+                                                          find_unused_parameters=True)
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
+                   args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1))
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {'input_ids':       batch[0],
+                      'attention_mask':  batch[1],
+                      'start_positions': batch[3],
+                      'end_positions':   batch[4]}
+            if args.model_type != 'distilbert':
+                inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2]
+            if args.model_type in ['xlnet', 'xlm']:
+                inputs.update({'cls_index': batch[5],
+                               'p_mask':       batch[6]})
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean() # mean() to average on multi-gpu parallel (not distributed) training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    if args.local_rank == -1 and args.evaluate_during_training:  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                        for key, value in results.items():
+                            tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
+                    tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step)
+                    logging_loss = tr_loss
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)
+
+    if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+        os.makedirs(args.output_dir)
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+    # Note that DistributedSampler samples randomly
+    eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset)
+    eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+    # Eval!
+    logger.info("***** Running evaluation {} *****".format(prefix))
+    logger.info("  Num examples = %d", len(dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    all_results = []
+    for batch in tqdm(eval_dataloader, desc="Evaluating"):
+        model.eval()
+        batch = tuple(t.to(args.device) for t in batch)
+        with torch.no_grad():
+            inputs = {'input_ids':      batch[0],
+                      'attention_mask': batch[1]
+                      }
+            if args.model_type != 'distilbert':
+                inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2]  # XLM don't use segment_ids
+            example_indices = batch[3]
+            if args.model_type in ['xlnet', 'xlm']:
+                inputs.update({'cls_index': batch[4],
+                               'p_mask':    batch[5]})
+            outputs = model(**inputs)
+
+        for i, example_index in enumerate(example_indices):
+            eval_feature = features[example_index.item()]
+            unique_id = int(eval_feature.unique_id)
+            if args.model_type in ['xlnet', 'xlm']:
+                # XLNet uses a more complex post-processing procedure
+                result = RawResultExtended(unique_id            = unique_id,
+                                           start_top_log_probs  = to_list(outputs[0][i]),
+                                           start_top_index      = to_list(outputs[1][i]),
+                                           end_top_log_probs    = to_list(outputs[2][i]),
+                                           end_top_index        = to_list(outputs[3][i]),
+                                           cls_logits           = to_list(outputs[4][i]))
+            else:
+                result = RawResult(unique_id    = unique_id,
+                                   start_logits = to_list(outputs[0][i]),
+                                   end_logits   = to_list(outputs[1][i]))
+            all_results.append(result)
+
+    # Compute predictions
+    output_prediction_file = os.path.join(args.output_dir, "predictions_{}.json".format(prefix))
+    output_nbest_file = os.path.join(args.output_dir, "nbest_predictions_{}.json".format(prefix))
+    if args.version_2_with_negative:
+        output_null_log_odds_file = os.path.join(args.output_dir, "null_odds_{}.json".format(prefix))
+    else:
+        output_null_log_odds_file = None
+
+    if args.model_type in ['xlnet', 'xlm']:
+        # XLNet uses a more complex post-processing procedure
+        write_predictions_extended(examples, features, all_results, args.n_best_size,
+                        args.max_answer_length, output_prediction_file,
+                        output_nbest_file, output_null_log_odds_file, args.predict_file,
+                        model.config.start_n_top, model.config.end_n_top,
+                        args.version_2_with_negative, tokenizer, args.verbose_logging)
+    else:
+        write_predictions(examples, features, all_results, args.n_best_size,
+                        args.max_answer_length, args.do_lower_case, output_prediction_file,
+                        output_nbest_file, output_null_log_odds_file, args.verbose_logging,
+                        args.version_2_with_negative, args.null_score_diff_threshold)
+
+    # Evaluate with the official SQuAD script
+    evaluate_options = EVAL_OPTS(data_file=args.predict_file,
+                                 pred_file=output_prediction_file,
+                                 na_prob_file=output_null_log_odds_file)
+    results = evaluate_on_squad(evaluate_options)
+    return results
+
+
+def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Load data features from cache or dataset file
+    input_file = args.predict_file if evaluate else args.train_file
+    cached_features_file = os.path.join(os.path.dirname(input_file), 'cached_{}_{}_{}'.format(
+        'dev' if evaluate else 'train',
+        list(filter(None, args.model_name_or_path.split('/'))).pop(),
+        str(args.max_seq_length)))
+    if os.path.exists(cached_features_file) and not args.overwrite_cache and not output_examples:
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", input_file)
+        examples = read_squad_examples(input_file=input_file,
+                                                is_training=not evaluate,
+                                                version_2_with_negative=args.version_2_with_negative)
+        features = convert_examples_to_features(examples=examples,
+                                                tokenizer=tokenizer,
+                                                max_seq_length=args.max_seq_length,
+                                                doc_stride=args.doc_stride,
+                                                max_query_length=args.max_query_length,
+                                                is_training=not evaluate)
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
+    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
+    all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long)
+    all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float)
+    if evaluate:
+        all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
+        dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
+                                all_example_index, all_cls_index, all_p_mask)
+    else:
+        all_start_positions = torch.tensor([f.start_position for f in features], dtype=torch.long)
+        all_end_positions = torch.tensor([f.end_position for f in features], dtype=torch.long)
+        dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
+                                all_start_positions, all_end_positions,
+                                all_cls_index, all_p_mask)
+
+    if output_examples:
+        return dataset, examples, features
+    return dataset
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    ## Required parameters
+    parser.add_argument("--train_file", default=None, type=str, required=True,
+                        help="SQuAD json for training. E.g., train-v1.1.json")
+    parser.add_argument("--predict_file", default=None, type=str, required=True,
+                        help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json")
+    parser.add_argument("--model_type", default=None, type=str, required=True,
+                        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
+                        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS))
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model checkpoints and predictions will be written.")
+
+    ## Other parameters
+    parser.add_argument("--config_name", default="", type=str,
+                        help="Pretrained config name or path if not the same as model_name")
+    parser.add_argument("--tokenizer_name", default="", type=str,
+                        help="Pretrained tokenizer name or path if not the same as model_name")
+    parser.add_argument("--cache_dir", default="", type=str,
+                        help="Where do you want to store the pre-trained models downloaded from s3")
+
+    parser.add_argument('--version_2_with_negative', action='store_true',
+                        help='If true, the SQuAD examples contain some that do not have an answer.')
+    parser.add_argument('--null_score_diff_threshold', type=float, default=0.0,
+                        help="If null_score - best_non_null is greater than the threshold predict null.")
+
+    parser.add_argument("--max_seq_length", default=384, type=int,
+                        help="The maximum total input sequence length after WordPiece tokenization. Sequences "
+                             "longer than this will be truncated, and sequences shorter than this will be padded.")
+    parser.add_argument("--doc_stride", default=128, type=int,
+                        help="When splitting up a long document into chunks, how much stride to take between chunks.")
+    parser.add_argument("--max_query_length", default=64, type=int,
+                        help="The maximum number of tokens for the question. Questions longer than this will "
+                             "be truncated to this length.")
+    parser.add_argument("--do_train", action='store_true',
+                        help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true',
+                        help="Whether to run eval on the dev set.")
+    parser.add_argument("--evaluate_during_training", action='store_true',
+                        help="Rul evaluation during training at each logging step.")
+    parser.add_argument("--do_lower_case", action='store_true',
+                        help="Set this flag if you are using an uncased model.")
+
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for evaluation.")
+    parser.add_argument("--learning_rate", default=5e-5, type=float,
+                        help="The initial learning rate for Adam.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument("--weight_decay", default=0.0, type=float,
+                        help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
+                        help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float,
+                        help="Max gradient norm.")
+    parser.add_argument("--num_train_epochs", default=3.0, type=float,
+                        help="Total number of training epochs to perform.")
+    parser.add_argument("--max_steps", default=-1, type=int,
+                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
+    parser.add_argument("--warmup_steps", default=0, type=int,
+                        help="Linear warmup over warmup_steps.")
+    parser.add_argument("--n_best_size", default=20, type=int,
+                        help="The total number of n-best predictions to generate in the nbest_predictions.json output file.")
+    parser.add_argument("--max_answer_length", default=30, type=int,
+                        help="The maximum length of an answer that can be generated. This is needed because the start "
+                             "and end predictions are not conditioned on one another.")
+    parser.add_argument("--verbose_logging", action='store_true',
+                        help="If true, all of the warnings related to data processing will be printed. "
+                             "A number of warnings are expected for a normal SQuAD evaluation.")
+
+    parser.add_argument('--logging_steps', type=int, default=50,
+                        help="Log every X updates steps.")
+    parser.add_argument('--save_steps', type=int, default=50,
+                        help="Save checkpoint every X updates steps.")
+    parser.add_argument("--eval_all_checkpoints", action='store_true',
+                        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
+    parser.add_argument("--no_cuda", action='store_true',
+                        help="Whether not to use CUDA when available")
+    parser.add_argument('--overwrite_output_dir', action='store_true',
+                        help="Overwrite the content of the output directory")
+    parser.add_argument('--overwrite_cache', action='store_true',
+                        help="Overwrite the cached training and evaluation sets")
+    parser.add_argument('--seed', type=int, default=42,
+                        help="random seed for initialization")
+
+    parser.add_argument("--local_rank", type=int, default=-1,
+                        help="local_rank for distributed training on gpus")
+    parser.add_argument('--fp16', action='store_true',
+                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
+    parser.add_argument('--fp16_opt_level', type=str, default='O1',
+                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                             "See details at https://nvidia.github.io/apex/amp.html")
+    parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
+    parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
+    args = parser.parse_args()
+
+    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
+        raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl')
+        args.n_gpu = 1
+    args.device = device
+
+    # Setup logging
+    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                        datefmt = '%m/%d/%Y %H:%M:%S',
+                        level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
+    logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+                    args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
+
+    # Set seed
+    set_seed(args)
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
+    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    # Before we do anything with models, we want to ensure that we get fp16 execution of torch.einsum if args.fp16 is set.
+    # Otherwise it'll default to "promote" mode, and we'll get fp32 operations. Note that running `--fp16_opt_level="O2"` will
+    # remove the need for this code, but it is still valid.
+    if args.fp16:
+        try:
+            import apex
+            apex.amp.register_half_function(torch, 'einsum')
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+
+    # Training
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+
+    # Save the trained model and the tokenizer
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model.to(args.device)
+
+
+    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))
+            logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN)  # Reduce model loading logs
+
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+
+        for checkpoint in checkpoints:
+            # Reload the model
+            global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
+            model = model_class.from_pretrained(checkpoint)
+            model.to(args.device)
+
+            # Evaluate
+            result = evaluate(args, model, tokenizer, prefix=global_step)
+
+            result = dict((k + ('_{}'.format(global_step) if global_step else ''), v) for k, v in result.items())
+            results.update(result)
+
+    logger.info("Results: {}".format(results))
+
+    return results
+
+
+if __name__ == "__main__":
+    main()
diff --git a/templates/adding_a_new_example_script/utils_xxx.py b/templates/adding_a_new_example_script/utils_xxx.py
new file mode 100644
index 0000000000..3f4145e028
--- /dev/null
+++ b/templates/adding_a_new_example_script/utils_xxx.py
@@ -0,0 +1,995 @@
+
+# coding=utf-8
+# Copyright 2018 XXX.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Load XXX dataset. """
+
+from __future__ import absolute_import, division, print_function
+
+import json
+import logging
+import math
+import collections
+from io import open
+
+from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
+
+# Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method)
+from utils_squad_evaluate import find_all_best_thresh_v2, make_qid_to_has_ans, get_raw_scores
+
+logger = logging.getLogger(__name__)
+
+
+class SquadExample(object):
+    """
+    A single training/test example for the Squad dataset.
+    For examples without an answer, the start and end position are -1.
+    """
+
+    def __init__(self,
+                 qas_id,
+                 question_text,
+                 doc_tokens,
+                 orig_answer_text=None,
+                 start_position=None,
+                 end_position=None,
+                 is_impossible=None):
+        self.qas_id = qas_id
+        self.question_text = question_text
+        self.doc_tokens = doc_tokens
+        self.orig_answer_text = orig_answer_text
+        self.start_position = start_position
+        self.end_position = end_position
+        self.is_impossible = is_impossible
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __repr__(self):
+        s = ""
+        s += "qas_id: %s" % (self.qas_id)
+        s += ", question_text: %s" % (
+            self.question_text)
+        s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens))
+        if self.start_position:
+            s += ", start_position: %d" % (self.start_position)
+        if self.end_position:
+            s += ", end_position: %d" % (self.end_position)
+        if self.is_impossible:
+            s += ", is_impossible: %r" % (self.is_impossible)
+        return s
+
+
+class InputFeatures(object):
+    """A single set of features of data."""
+
+    def __init__(self,
+                 unique_id,
+                 example_index,
+                 doc_span_index,
+                 tokens,
+                 token_to_orig_map,
+                 token_is_max_context,
+                 input_ids,
+                 input_mask,
+                 segment_ids,
+                 cls_index,
+                 p_mask,
+                 paragraph_len,
+                 start_position=None,
+                 end_position=None,
+                 is_impossible=None):
+        self.unique_id = unique_id
+        self.example_index = example_index
+        self.doc_span_index = doc_span_index
+        self.tokens = tokens
+        self.token_to_orig_map = token_to_orig_map
+        self.token_is_max_context = token_is_max_context
+        self.input_ids = input_ids
+        self.input_mask = input_mask
+        self.segment_ids = segment_ids
+        self.cls_index = cls_index
+        self.p_mask = p_mask
+        self.paragraph_len = paragraph_len
+        self.start_position = start_position
+        self.end_position = end_position
+        self.is_impossible = is_impossible
+
+
+def read_squad_examples(input_file, is_training, version_2_with_negative):
+    """Read a SQuAD json file into a list of SquadExample."""
+    with open(input_file, "r", encoding='utf-8') as reader:
+        input_data = json.load(reader)["data"]
+
+    def is_whitespace(c):
+        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
+            return True
+        return False
+
+    examples = []
+    for entry in input_data:
+        for paragraph in entry["paragraphs"]:
+            paragraph_text = paragraph["context"]
+            doc_tokens = []
+            char_to_word_offset = []
+            prev_is_whitespace = True
+            for c in paragraph_text:
+                if is_whitespace(c):
+                    prev_is_whitespace = True
+                else:
+                    if prev_is_whitespace:
+                        doc_tokens.append(c)
+                    else:
+                        doc_tokens[-1] += c
+                    prev_is_whitespace = False
+                char_to_word_offset.append(len(doc_tokens) - 1)
+
+            for qa in paragraph["qas"]:
+                qas_id = qa["id"]
+                question_text = qa["question"]
+                start_position = None
+                end_position = None
+                orig_answer_text = None
+                is_impossible = False
+                if is_training:
+                    if version_2_with_negative:
+                        is_impossible = qa["is_impossible"]
+                    if (len(qa["answers"]) != 1) and (not is_impossible):
+                        raise ValueError(
+                            "For training, each question should have exactly 1 answer.")
+                    if not is_impossible:
+                        answer = qa["answers"][0]
+                        orig_answer_text = answer["text"]
+                        answer_offset = answer["answer_start"]
+                        answer_length = len(orig_answer_text)
+                        start_position = char_to_word_offset[answer_offset]
+                        end_position = char_to_word_offset[answer_offset + answer_length - 1]
+                        # Only add answers where the text can be exactly recovered from the
+                        # document. If this CAN'T happen it's likely due to weird Unicode
+                        # stuff so we will just skip the example.
+                        #
+                        # Note that this means for training mode, every example is NOT
+                        # guaranteed to be preserved.
+                        actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
+                        cleaned_answer_text = " ".join(
+                            whitespace_tokenize(orig_answer_text))
+                        if actual_text.find(cleaned_answer_text) == -1:
+                            logger.warning("Could not find answer: '%s' vs. '%s'",
+                                           actual_text, cleaned_answer_text)
+                            continue
+                    else:
+                        start_position = -1
+                        end_position = -1
+                        orig_answer_text = ""
+
+                example = SquadExample(
+                    qas_id=qas_id,
+                    question_text=question_text,
+                    doc_tokens=doc_tokens,
+                    orig_answer_text=orig_answer_text,
+                    start_position=start_position,
+                    end_position=end_position,
+                    is_impossible=is_impossible)
+                examples.append(example)
+    return examples
+
+
+def convert_examples_to_features(examples, tokenizer, max_seq_length,
+                                 doc_stride, max_query_length, is_training,
+                                 cls_token_at_end=False,
+                                 cls_token='[CLS]', sep_token='[SEP]', pad_token=0,
+                                 sequence_a_segment_id=0, sequence_b_segment_id=1,
+                                 cls_token_segment_id=0, pad_token_segment_id=0,
+                                 mask_padding_with_zero=True):
+    """Loads a data file into a list of `InputBatch`s."""
+
+    unique_id = 1000000000
+    # cnt_pos, cnt_neg = 0, 0
+    # max_N, max_M = 1024, 1024
+    # f = np.zeros((max_N, max_M), dtype=np.float32)
+
+    features = []
+    for (example_index, example) in enumerate(examples):
+
+        # if example_index % 100 == 0:
+        #     logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)
+
+        query_tokens = tokenizer.tokenize(example.question_text)
+
+        if len(query_tokens) > max_query_length:
+            query_tokens = query_tokens[0:max_query_length]
+
+        tok_to_orig_index = []
+        orig_to_tok_index = []
+        all_doc_tokens = []
+        for (i, token) in enumerate(example.doc_tokens):
+            orig_to_tok_index.append(len(all_doc_tokens))
+            sub_tokens = tokenizer.tokenize(token)
+            for sub_token in sub_tokens:
+                tok_to_orig_index.append(i)
+                all_doc_tokens.append(sub_token)
+
+        tok_start_position = None
+        tok_end_position = None
+        if is_training and example.is_impossible:
+            tok_start_position = -1
+            tok_end_position = -1
+        if is_training and not example.is_impossible:
+            tok_start_position = orig_to_tok_index[example.start_position]
+            if example.end_position < len(example.doc_tokens) - 1:
+                tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
+            else:
+                tok_end_position = len(all_doc_tokens) - 1
+            (tok_start_position, tok_end_position) = _improve_answer_span(
+                all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
+                example.orig_answer_text)
+
+        # The -3 accounts for [CLS], [SEP] and [SEP]
+        max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
+
+        # We can have documents that are longer than the maximum sequence length.
+        # To deal with this we do a sliding window approach, where we take chunks
+        # of the up to our max length with a stride of `doc_stride`.
+        _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
+            "DocSpan", ["start", "length"])
+        doc_spans = []
+        start_offset = 0
+        while start_offset < len(all_doc_tokens):
+            length = len(all_doc_tokens) - start_offset
+            if length > max_tokens_for_doc:
+                length = max_tokens_for_doc
+            doc_spans.append(_DocSpan(start=start_offset, length=length))
+            if start_offset + length == len(all_doc_tokens):
+                break
+            start_offset += min(length, doc_stride)
+
+        for (doc_span_index, doc_span) in enumerate(doc_spans):
+            tokens = []
+            token_to_orig_map = {}
+            token_is_max_context = {}
+            segment_ids = []
+
+            # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer)
+            # Original TF implem also keep the classification token (set to 0) (not sure why...)
+            p_mask = []
+
+            # CLS token at the beginning
+            if not cls_token_at_end:
+                tokens.append(cls_token)
+                segment_ids.append(cls_token_segment_id)
+                p_mask.append(0)
+                cls_index = 0
+
+            # Query
+            for token in query_tokens:
+                tokens.append(token)
+                segment_ids.append(sequence_a_segment_id)
+                p_mask.append(1)
+
+            # SEP token
+            tokens.append(sep_token)
+            segment_ids.append(sequence_a_segment_id)
+            p_mask.append(1)
+
+            # Paragraph
+            for i in range(doc_span.length):
+                split_token_index = doc_span.start + i
+                token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]
+
+                is_max_context = _check_is_max_context(doc_spans, doc_span_index,
+                                                       split_token_index)
+                token_is_max_context[len(tokens)] = is_max_context
+                tokens.append(all_doc_tokens[split_token_index])
+                segment_ids.append(sequence_b_segment_id)
+                p_mask.append(0)
+            paragraph_len = doc_span.length
+
+            # SEP token
+            tokens.append(sep_token)
+            segment_ids.append(sequence_b_segment_id)
+            p_mask.append(1)
+
+            # CLS token at the end
+            if cls_token_at_end:
+                tokens.append(cls_token)
+                segment_ids.append(cls_token_segment_id)
+                p_mask.append(0)
+                cls_index = len(tokens) - 1  # Index of classification token
+
+            input_ids = tokenizer.convert_tokens_to_ids(tokens)
+
+            # The mask has 1 for real tokens and 0 for padding tokens. Only real
+            # tokens are attended to.
+            input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
+
+            # Zero-pad up to the sequence length.
+            while len(input_ids) < max_seq_length:
+                input_ids.append(pad_token)
+                input_mask.append(0 if mask_padding_with_zero else 1)
+                segment_ids.append(pad_token_segment_id)
+                p_mask.append(1)
+
+            assert len(input_ids) == max_seq_length
+            assert len(input_mask) == max_seq_length
+            assert len(segment_ids) == max_seq_length
+
+            span_is_impossible = example.is_impossible
+            start_position = None
+            end_position = None
+            if is_training and not span_is_impossible:
+                # For training, if our document chunk does not contain an annotation
+                # we throw it out, since there is nothing to predict.
+                doc_start = doc_span.start
+                doc_end = doc_span.start + doc_span.length - 1
+                out_of_span = False
+                if not (tok_start_position >= doc_start and
+                        tok_end_position <= doc_end):
+                    out_of_span = True
+                if out_of_span:
+                    start_position = 0
+                    end_position = 0
+                    span_is_impossible = True
+                else:
+                    doc_offset = len(query_tokens) + 2
+                    start_position = tok_start_position - doc_start + doc_offset
+                    end_position = tok_end_position - doc_start + doc_offset
+
+            if is_training and span_is_impossible:
+                start_position = cls_index
+                end_position = cls_index
+
+            if example_index < 20:
+                logger.info("*** Example ***")
+                logger.info("unique_id: %s" % (unique_id))
+                logger.info("example_index: %s" % (example_index))
+                logger.info("doc_span_index: %s" % (doc_span_index))
+                logger.info("tokens: %s" % " ".join(tokens))
+                logger.info("token_to_orig_map: %s" % " ".join([
+                    "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()]))
+                logger.info("token_is_max_context: %s" % " ".join([
+                    "%d:%s" % (x, y) for (x, y) in token_is_max_context.items()
+                ]))
+                logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
+                logger.info(
+                    "input_mask: %s" % " ".join([str(x) for x in input_mask]))
+                logger.info(
+                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
+                if is_training and span_is_impossible:
+                    logger.info("impossible example")
+                if is_training and not span_is_impossible:
+                    answer_text = " ".join(tokens[start_position:(end_position + 1)])
+                    logger.info("start_position: %d" % (start_position))
+                    logger.info("end_position: %d" % (end_position))
+                    logger.info(
+                        "answer: %s" % (answer_text))
+
+            features.append(
+                InputFeatures(
+                    unique_id=unique_id,
+                    example_index=example_index,
+                    doc_span_index=doc_span_index,
+                    tokens=tokens,
+                    token_to_orig_map=token_to_orig_map,
+                    token_is_max_context=token_is_max_context,
+                    input_ids=input_ids,
+                    input_mask=input_mask,
+                    segment_ids=segment_ids,
+                    cls_index=cls_index,
+                    p_mask=p_mask,
+                    paragraph_len=paragraph_len,
+                    start_position=start_position,
+                    end_position=end_position,
+                    is_impossible=span_is_impossible))
+            unique_id += 1
+
+    return features
+
+
+def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer,
+                         orig_answer_text):
+    """Returns tokenized answer spans that better match the annotated answer."""
+
+    # The SQuAD annotations are character based. We first project them to
+    # whitespace-tokenized words. But then after WordPiece tokenization, we can
+    # often find a "better match". For example:
+    #
+    #   Question: What year was John Smith born?
+    #   Context: The leader was John Smith (1895-1943).
+    #   Answer: 1895
+    #
+    # The original whitespace-tokenized answer will be "(1895-1943).". However
+    # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match
+    # the exact answer, 1895.
+    #
+    # However, this is not always possible. Consider the following:
+    #
+    #   Question: What country is the top exporter of electornics?
+    #   Context: The Japanese electronics industry is the lagest in the world.
+    #   Answer: Japan
+    #
+    # In this case, the annotator chose "Japan" as a character sub-span of
+    # the word "Japanese". Since our WordPiece tokenizer does not split
+    # "Japanese", we just use "Japanese" as the annotation. This is fairly rare
+    # in SQuAD, but does happen.
+    tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text))
+
+    for new_start in range(input_start, input_end + 1):
+        for new_end in range(input_end, new_start - 1, -1):
+            text_span = " ".join(doc_tokens[new_start:(new_end + 1)])
+            if text_span == tok_answer_text:
+                return (new_start, new_end)
+
+    return (input_start, input_end)
+
+
+def _check_is_max_context(doc_spans, cur_span_index, position):
+    """Check if this is the 'max context' doc span for the token."""
+
+    # Because of the sliding window approach taken to scoring documents, a single
+    # token can appear in multiple documents. E.g.
+    #  Doc: the man went to the store and bought a gallon of milk
+    #  Span A: the man went to the
+    #  Span B: to the store and bought
+    #  Span C: and bought a gallon of
+    #  ...
+    #
+    # Now the word 'bought' will have two scores from spans B and C. We only
+    # want to consider the score with "maximum context", which we define as
+    # the *minimum* of its left and right context (the *sum* of left and
+    # right context will always be the same, of course).
+    #
+    # In the example the maximum context for 'bought' would be span C since
+    # it has 1 left context and 3 right context, while span B has 4 left context
+    # and 0 right context.
+    best_score = None
+    best_span_index = None
+    for (span_index, doc_span) in enumerate(doc_spans):
+        end = doc_span.start + doc_span.length - 1
+        if position < doc_span.start:
+            continue
+        if position > end:
+            continue
+        num_left_context = position - doc_span.start
+        num_right_context = end - position
+        score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
+        if best_score is None or score > best_score:
+            best_score = score
+            best_span_index = span_index
+
+    return cur_span_index == best_span_index
+
+
+RawResult = collections.namedtuple("RawResult",
+                                   ["unique_id", "start_logits", "end_logits"])
+
+def write_predictions(all_examples, all_features, all_results, n_best_size,
+                      max_answer_length, do_lower_case, output_prediction_file,
+                      output_nbest_file, output_null_log_odds_file, verbose_logging,
+                      version_2_with_negative, null_score_diff_threshold):
+    """Write final predictions to the json file and log-odds of null if needed."""
+    logger.info("Writing predictions to: %s" % (output_prediction_file))
+    logger.info("Writing nbest to: %s" % (output_nbest_file))
+
+    example_index_to_features = collections.defaultdict(list)
+    for feature in all_features:
+        example_index_to_features[feature.example_index].append(feature)
+
+    unique_id_to_result = {}
+    for result in all_results:
+        unique_id_to_result[result.unique_id] = result
+
+    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
+        "PrelimPrediction",
+        ["feature_index", "start_index", "end_index", "start_logit", "end_logit"])
+
+    all_predictions = collections.OrderedDict()
+    all_nbest_json = collections.OrderedDict()
+    scores_diff_json = collections.OrderedDict()
+
+    for (example_index, example) in enumerate(all_examples):
+        features = example_index_to_features[example_index]
+
+        prelim_predictions = []
+        # keep track of the minimum score of null start+end of position 0
+        score_null = 1000000  # large and positive
+        min_null_feature_index = 0  # the paragraph slice with min null score
+        null_start_logit = 0  # the start logit at the slice with min null score
+        null_end_logit = 0  # the end logit at the slice with min null score
+        for (feature_index, feature) in enumerate(features):
+            result = unique_id_to_result[feature.unique_id]
+            start_indexes = _get_best_indexes(result.start_logits, n_best_size)
+            end_indexes = _get_best_indexes(result.end_logits, n_best_size)
+            # if we could have irrelevant answers, get the min score of irrelevant
+            if version_2_with_negative:
+                feature_null_score = result.start_logits[0] + result.end_logits[0]
+                if feature_null_score < score_null:
+                    score_null = feature_null_score
+                    min_null_feature_index = feature_index
+                    null_start_logit = result.start_logits[0]
+                    null_end_logit = result.end_logits[0]
+            for start_index in start_indexes:
+                for end_index in end_indexes:
+                    # We could hypothetically create invalid predictions, e.g., predict
+                    # that the start of the span is in the question. We throw out all
+                    # invalid predictions.
+                    if start_index >= len(feature.tokens):
+                        continue
+                    if end_index >= len(feature.tokens):
+                        continue
+                    if start_index not in feature.token_to_orig_map:
+                        continue
+                    if end_index not in feature.token_to_orig_map:
+                        continue
+                    if not feature.token_is_max_context.get(start_index, False):
+                        continue
+                    if end_index < start_index:
+                        continue
+                    length = end_index - start_index + 1
+                    if length > max_answer_length:
+                        continue
+                    prelim_predictions.append(
+                        _PrelimPrediction(
+                            feature_index=feature_index,
+                            start_index=start_index,
+                            end_index=end_index,
+                            start_logit=result.start_logits[start_index],
+                            end_logit=result.end_logits[end_index]))
+        if version_2_with_negative:
+            prelim_predictions.append(
+                _PrelimPrediction(
+                    feature_index=min_null_feature_index,
+                    start_index=0,
+                    end_index=0,
+                    start_logit=null_start_logit,
+                    end_logit=null_end_logit))
+        prelim_predictions = sorted(
+            prelim_predictions,
+            key=lambda x: (x.start_logit + x.end_logit),
+            reverse=True)
+
+        _NbestPrediction = collections.namedtuple(  # pylint: disable=invalid-name
+            "NbestPrediction", ["text", "start_logit", "end_logit"])
+
+        seen_predictions = {}
+        nbest = []
+        for pred in prelim_predictions:
+            if len(nbest) >= n_best_size:
+                break
+            feature = features[pred.feature_index]
+            if pred.start_index > 0:  # this is a non-null prediction
+                tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)]
+                orig_doc_start = feature.token_to_orig_map[pred.start_index]
+                orig_doc_end = feature.token_to_orig_map[pred.end_index]
+                orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)]
+                tok_text = " ".join(tok_tokens)
+
+                # De-tokenize WordPieces that have been split off.
+                tok_text = tok_text.replace(" ##", "")
+                tok_text = tok_text.replace("##", "")
+
+                # Clean whitespace
+                tok_text = tok_text.strip()
+                tok_text = " ".join(tok_text.split())
+                orig_text = " ".join(orig_tokens)
+
+                final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging)
+                if final_text in seen_predictions:
+                    continue
+
+                seen_predictions[final_text] = True
+            else:
+                final_text = ""
+                seen_predictions[final_text] = True
+
+            nbest.append(
+                _NbestPrediction(
+                    text=final_text,
+                    start_logit=pred.start_logit,
+                    end_logit=pred.end_logit))
+        # if we didn't include the empty option in the n-best, include it
+        if version_2_with_negative:
+            if "" not in seen_predictions:
+                nbest.append(
+                    _NbestPrediction(
+                        text="",
+                        start_logit=null_start_logit,
+                        end_logit=null_end_logit))
+                
+            # In very rare edge cases we could only have single null prediction.
+            # So we just create a nonce prediction in this case to avoid failure.
+            if len(nbest)==1:
+                nbest.insert(0,
+                    _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))
+
+        # In very rare edge cases we could have no valid predictions. So we
+        # just create a nonce prediction in this case to avoid failure.
+        if not nbest:
+            nbest.append(
+                _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))
+
+        assert len(nbest) >= 1
+
+        total_scores = []
+        best_non_null_entry = None
+        for entry in nbest:
+            total_scores.append(entry.start_logit + entry.end_logit)
+            if not best_non_null_entry:
+                if entry.text:
+                    best_non_null_entry = entry
+
+        probs = _compute_softmax(total_scores)
+
+        nbest_json = []
+        for (i, entry) in enumerate(nbest):
+            output = collections.OrderedDict()
+            output["text"] = entry.text
+            output["probability"] = probs[i]
+            output["start_logit"] = entry.start_logit
+            output["end_logit"] = entry.end_logit
+            nbest_json.append(output)
+
+        assert len(nbest_json) >= 1
+
+        if not version_2_with_negative:
+            all_predictions[example.qas_id] = nbest_json[0]["text"]
+        else:
+            # predict "" iff the null score - the score of best non-null > threshold
+            score_diff = score_null - best_non_null_entry.start_logit - (
+                best_non_null_entry.end_logit)
+            scores_diff_json[example.qas_id] = score_diff
+            if score_diff > null_score_diff_threshold:
+                all_predictions[example.qas_id] = ""
+            else:
+                all_predictions[example.qas_id] = best_non_null_entry.text
+        all_nbest_json[example.qas_id] = nbest_json
+
+    with open(output_prediction_file, "w") as writer:
+        writer.write(json.dumps(all_predictions, indent=4) + "\n")
+
+    with open(output_nbest_file, "w") as writer:
+        writer.write(json.dumps(all_nbest_json, indent=4) + "\n")
+
+    if version_2_with_negative:
+        with open(output_null_log_odds_file, "w") as writer:
+            writer.write(json.dumps(scores_diff_json, indent=4) + "\n")
+
+    return all_predictions
+
+
+# For XLNet (and XLM which uses the same head)
+RawResultExtended = collections.namedtuple("RawResultExtended",
+    ["unique_id", "start_top_log_probs", "start_top_index",
+     "end_top_log_probs", "end_top_index", "cls_logits"])
+
+
+def write_predictions_extended(all_examples, all_features, all_results, n_best_size,
+                                max_answer_length, output_prediction_file,
+                                output_nbest_file,
+                                output_null_log_odds_file, orig_data_file,
+                                start_n_top, end_n_top, version_2_with_negative,
+                                tokenizer, verbose_logging):
+    """ XLNet write prediction logic (more complex than Bert's).
+        Write final predictions to the json file and log-odds of null if needed.
+
+        Requires utils_squad_evaluate.py
+    """
+    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
+        "PrelimPrediction",
+        ["feature_index", "start_index", "end_index",
+        "start_log_prob", "end_log_prob"])
+
+    _NbestPrediction = collections.namedtuple(  # pylint: disable=invalid-name
+        "NbestPrediction", ["text", "start_log_prob", "end_log_prob"])
+
+    logger.info("Writing predictions to: %s", output_prediction_file)
+    # logger.info("Writing nbest to: %s" % (output_nbest_file))
+
+    example_index_to_features = collections.defaultdict(list)
+    for feature in all_features:
+        example_index_to_features[feature.example_index].append(feature)
+
+    unique_id_to_result = {}
+    for result in all_results:
+        unique_id_to_result[result.unique_id] = result
+
+    all_predictions = collections.OrderedDict()
+    all_nbest_json = collections.OrderedDict()
+    scores_diff_json = collections.OrderedDict()
+
+    for (example_index, example) in enumerate(all_examples):
+        features = example_index_to_features[example_index]
+
+        prelim_predictions = []
+        # keep track of the minimum score of null start+end of position 0
+        score_null = 1000000  # large and positive
+
+        for (feature_index, feature) in enumerate(features):
+            result = unique_id_to_result[feature.unique_id]
+
+            cur_null_score = result.cls_logits
+
+            # if we could have irrelevant answers, get the min score of irrelevant
+            score_null = min(score_null, cur_null_score)
+
+            for i in range(start_n_top):
+                for j in range(end_n_top):
+                    start_log_prob = result.start_top_log_probs[i]
+                    start_index = result.start_top_index[i]
+
+                    j_index = i * end_n_top + j
+
+                    end_log_prob = result.end_top_log_probs[j_index]
+                    end_index = result.end_top_index[j_index]
+
+                    # We could hypothetically create invalid predictions, e.g., predict
+                    # that the start of the span is in the question. We throw out all
+                    # invalid predictions.
+                    if start_index >= feature.paragraph_len - 1:
+                        continue
+                    if end_index >= feature.paragraph_len - 1:
+                        continue
+
+                    if not feature.token_is_max_context.get(start_index, False):
+                        continue
+                    if end_index < start_index:
+                        continue
+                    length = end_index - start_index + 1
+                    if length > max_answer_length:
+                        continue
+
+                    prelim_predictions.append(
+                        _PrelimPrediction(
+                            feature_index=feature_index,
+                            start_index=start_index,
+                            end_index=end_index,
+                            start_log_prob=start_log_prob,
+                            end_log_prob=end_log_prob))
+
+        prelim_predictions = sorted(
+            prelim_predictions,
+            key=lambda x: (x.start_log_prob + x.end_log_prob),
+            reverse=True)
+
+        seen_predictions = {}
+        nbest = []
+        for pred in prelim_predictions:
+            if len(nbest) >= n_best_size:
+                break
+            feature = features[pred.feature_index]
+
+            # XLNet un-tokenizer
+            # Let's keep it simple for now and see if we need all this later.
+            # 
+            # tok_start_to_orig_index = feature.tok_start_to_orig_index
+            # tok_end_to_orig_index = feature.tok_end_to_orig_index
+            # start_orig_pos = tok_start_to_orig_index[pred.start_index]
+            # end_orig_pos = tok_end_to_orig_index[pred.end_index]
+            # paragraph_text = example.paragraph_text
+            # final_text = paragraph_text[start_orig_pos: end_orig_pos + 1].strip()
+
+            # Previously used Bert untokenizer
+            tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)]
+            orig_doc_start = feature.token_to_orig_map[pred.start_index]
+            orig_doc_end = feature.token_to_orig_map[pred.end_index]
+            orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)]
+            tok_text = tokenizer.convert_tokens_to_string(tok_tokens)
+
+            # Clean whitespace
+            tok_text = tok_text.strip()
+            tok_text = " ".join(tok_text.split())
+            orig_text = " ".join(orig_tokens)
+
+            final_text = get_final_text(tok_text, orig_text, tokenizer.do_lower_case,
+                                        verbose_logging)
+
+            if final_text in seen_predictions:
+                continue
+
+            seen_predictions[final_text] = True
+
+            nbest.append(
+                _NbestPrediction(
+                    text=final_text,
+                    start_log_prob=pred.start_log_prob,
+                    end_log_prob=pred.end_log_prob))
+
+        # In very rare edge cases we could have no valid predictions. So we
+        # just create a nonce prediction in this case to avoid failure.
+        if not nbest:
+            nbest.append(
+                _NbestPrediction(text="", start_log_prob=-1e6,
+                end_log_prob=-1e6))
+
+        total_scores = []
+        best_non_null_entry = None
+        for entry in nbest:
+            total_scores.append(entry.start_log_prob + entry.end_log_prob)
+            if not best_non_null_entry:
+                best_non_null_entry = entry
+
+        probs = _compute_softmax(total_scores)
+
+        nbest_json = []
+        for (i, entry) in enumerate(nbest):
+            output = collections.OrderedDict()
+            output["text"] = entry.text
+            output["probability"] = probs[i]
+            output["start_log_prob"] = entry.start_log_prob
+            output["end_log_prob"] = entry.end_log_prob
+            nbest_json.append(output)
+
+        assert len(nbest_json) >= 1
+        assert best_non_null_entry is not None
+
+        score_diff = score_null
+        scores_diff_json[example.qas_id] = score_diff
+        # note(zhiliny): always predict best_non_null_entry
+        # and the evaluation script will search for the best threshold
+        all_predictions[example.qas_id] = best_non_null_entry.text
+
+        all_nbest_json[example.qas_id] = nbest_json
+
+    with open(output_prediction_file, "w") as writer:
+        writer.write(json.dumps(all_predictions, indent=4) + "\n")
+
+    with open(output_nbest_file, "w") as writer:
+        writer.write(json.dumps(all_nbest_json, indent=4) + "\n")
+
+    if version_2_with_negative:
+        with open(output_null_log_odds_file, "w") as writer:
+            writer.write(json.dumps(scores_diff_json, indent=4) + "\n")
+
+    with open(orig_data_file, "r", encoding='utf-8') as reader:
+        orig_data = json.load(reader)["data"]
+
+    qid_to_has_ans = make_qid_to_has_ans(orig_data)
+    has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]
+    no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]
+    exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions)
+    out_eval = {}
+
+    find_all_best_thresh_v2(out_eval, all_predictions, exact_raw, f1_raw, scores_diff_json, qid_to_has_ans)
+
+    return out_eval
+
+
+def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False):
+    """Project the tokenized prediction back to the original text."""
+
+    # When we created the data, we kept track of the alignment between original
+    # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So
+    # now `orig_text` contains the span of our original text corresponding to the
+    # span that we predicted.
+    #
+    # However, `orig_text` may contain extra characters that we don't want in
+    # our prediction.
+    #
+    # For example, let's say:
+    #   pred_text = steve smith
+    #   orig_text = Steve Smith's
+    #
+    # We don't want to return `orig_text` because it contains the extra "'s".
+    #
+    # We don't want to return `pred_text` because it's already been normalized
+    # (the SQuAD eval script also does punctuation stripping/lower casing but
+    # our tokenizer does additional normalization like stripping accent
+    # characters).
+    #
+    # What we really want to return is "Steve Smith".
+    #
+    # Therefore, we have to apply a semi-complicated alignment heuristic between
+    # `pred_text` and `orig_text` to get a character-to-character alignment. This
+    # can fail in certain cases in which case we just return `orig_text`.
+
+    def _strip_spaces(text):
+        ns_chars = []
+        ns_to_s_map = collections.OrderedDict()
+        for (i, c) in enumerate(text):
+            if c == " ":
+                continue
+            ns_to_s_map[len(ns_chars)] = i
+            ns_chars.append(c)
+        ns_text = "".join(ns_chars)
+        return (ns_text, ns_to_s_map)
+
+    # We first tokenize `orig_text`, strip whitespace from the result
+    # and `pred_text`, and check if they are the same length. If they are
+    # NOT the same length, the heuristic has failed. If they are the same
+    # length, we assume the characters are one-to-one aligned.
+    tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
+
+    tok_text = " ".join(tokenizer.tokenize(orig_text))
+
+    start_position = tok_text.find(pred_text)
+    if start_position == -1:
+        if verbose_logging:
+            logger.info(
+                "Unable to find text: '%s' in '%s'" % (pred_text, orig_text))
+        return orig_text
+    end_position = start_position + len(pred_text) - 1
+
+    (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text)
+    (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text)
+
+    if len(orig_ns_text) != len(tok_ns_text):
+        if verbose_logging:
+            logger.info("Length not equal after stripping spaces: '%s' vs '%s'",
+                        orig_ns_text, tok_ns_text)
+        return orig_text
+
+    # We then project the characters in `pred_text` back to `orig_text` using
+    # the character-to-character alignment.
+    tok_s_to_ns_map = {}
+    for (i, tok_index) in tok_ns_to_s_map.items():
+        tok_s_to_ns_map[tok_index] = i
+
+    orig_start_position = None
+    if start_position in tok_s_to_ns_map:
+        ns_start_position = tok_s_to_ns_map[start_position]
+        if ns_start_position in orig_ns_to_s_map:
+            orig_start_position = orig_ns_to_s_map[ns_start_position]
+
+    if orig_start_position is None:
+        if verbose_logging:
+            logger.info("Couldn't map start position")
+        return orig_text
+
+    orig_end_position = None
+    if end_position in tok_s_to_ns_map:
+        ns_end_position = tok_s_to_ns_map[end_position]
+        if ns_end_position in orig_ns_to_s_map:
+            orig_end_position = orig_ns_to_s_map[ns_end_position]
+
+    if orig_end_position is None:
+        if verbose_logging:
+            logger.info("Couldn't map end position")
+        return orig_text
+
+    output_text = orig_text[orig_start_position:(orig_end_position + 1)]
+    return output_text
+
+
+def _get_best_indexes(logits, n_best_size):
+    """Get the n-best logits from a list."""
+    index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True)
+
+    best_indexes = []
+    for i in range(len(index_and_score)):
+        if i >= n_best_size:
+            break
+        best_indexes.append(index_and_score[i][0])
+    return best_indexes
+
+
+def _compute_softmax(scores):
+    """Compute softmax probability over raw logits."""
+    if not scores:
+        return []
+
+    max_score = None
+    for score in scores:
+        if max_score is None or score > max_score:
+            max_score = score
+
+    exp_scores = []
+    total_sum = 0.0
+    for score in scores:
+        x = math.exp(score - max_score)
+        exp_scores.append(x)
+        total_sum += x
+
+    probs = []
+    for score in exp_scores:
+        probs.append(score / total_sum)
+    return probs
diff --git a/templates/adding_a_new_model/README.md b/templates/adding_a_new_model/README.md
new file mode 100644
index 0000000000..1569b51e89
--- /dev/null
+++ b/templates/adding_a_new_model/README.md
@@ -0,0 +1,62 @@
+# How to add a new model in 🤗Transformers
+
+This folder describes the process to add a new model in 🤗Transformers and provide templates for the required files.
+
+The library is designed to incorporate a variety of models and code bases. As such the process for adding a new model usually mostly consists in copy-pasting to relevant original code in the various sections of the templates included in the present repository.
+
+One important point though is that the library has the following goals impacting the way models are incorporated:
+
+- one specific feature of the API is the capability to run the model and tokenizer inline. The tokenization code thus often have to be slightly adapted to allow for running in the python interpreter.
+- the package is also designed to be as self-consistent and with a small and reliable set of packages dependencies. In consequence, additional dependencies are usually not allowed when adding a model but can be allowed for the inclusion of a new tokenizer (recent examples of dependencies added for tokenizer specificites includes `sentencepiece` and `sacremoses`). Please make sure to check the existing dependencies when possible before adding a new one.
+
+For a quick overview of the library organization, please check the [QuickStart section of the documentation](https://huggingface.co/transformers/quickstart.html).
+
+# Typical workflow for including a model
+
+Here an overview of the general workflow: 
+
+- [ ] add model/configuration/tokenization classes
+- [ ] add conversion scripts
+- [ ] add tests
+- [ ] finalize
+
+Let's details what should be done at each step
+
+## Adding model/configuration/tokenization classes
+
+Here is the workflow for adding model/configuration/tokenization classes:
+
+- [ ] copy the python files from the present folder to the main folder and rename them, replacing `xxx` with your model name,
+- [ ] edit the files to replace `XXX` (with various casing) with your model name
+- [ ] copy-past or create a simple configuration class for your model in the `configuration_...` file
+- [ ] copy-past or create the code for your model in the `modeling_...` files (PyTorch and TF 2.0)
+- [ ] copy-past or create a tokenizer class for your model in the `tokenization_...` file
+
+# Adding conversion scripts
+
+Here is the workflow for the conversion scripts:
+
+- [ ] copy the conversion script (`convert_...`) from the present folder to the main folder.
+- [ ] edit this scipt to convert your original checkpoint weights to the current pytorch ones.
+
+# Adding tests:
+
+Here is the workflow for the adding tests:
+
+- [ ] copy the python files from the `tests` sub-folder of the present folder to the `tests` subfolder of the main folder and rename them, replacing `xxx` with your model name,
+- [ ] edit the tests files to replace `XXX` (with various casing) with your model name
+- [ ] edit the tests code as needed
+
+# Final steps
+
+You can then finish the addition step by adding imports for your classes in the common files:
+
+- [ ] add import for all the relevant classes in `__init__.py`
+- [ ] add your configuration in `configuration_auto.py`
+- [ ] add your PyTorch and TF 2.0 model respectively in `modeling_auto.py` and `modeling_tf_auto.py`
+- [ ] add your tokenizer in `tokenization_auto.py`
+- [ ] add your models and tokenizer to `pipeline.py`
+- [ ] add a link to your conversion script in the main conversion utility (currently in `__main__` but will be moved to the `commands` subfolder in the near future)
+- [ ] edit the PyTorch to TF 2.0 conversion script to add your model in the `convert_pytorch_checkpoint_to_tf2.py` file
+- [ ] add a mention of your model in the doc: `README.md` and the documentation it-self at `docs/source/pretrained_models.rst`.
+- [ ] upload the pretrained weigths, configurations and vocabulary files.
diff --git a/templates/adding_a_new_model/configuration_xxx.py b/templates/adding_a_new_model/configuration_xxx.py
new file mode 100644
index 0000000000..b1614e71af
--- /dev/null
+++ b/templates/adding_a_new_model/configuration_xxx.py
@@ -0,0 +1,130 @@
+# coding=utf-8
+# Copyright 2010, XXX authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" XXX model configuration """
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import logging
+import sys
+import six
+from io import open
+
+from .configuration_utils import PretrainedConfig
+
+logger = logging.getLogger(__name__)
+
+XXX_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    'xxx-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-base-uncased-config.json",
+    'xxx-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-large-uncased-config.json",
+}
+
+
+class XxxConfig(PretrainedConfig):
+    r"""
+        :class:`~transformers.XxxConfig` is the configuration class to store the configuration of a
+        `XxxModel`.
+
+
+        Arguments:
+            vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XxxModel`.
+            hidden_size: Size of the encoder layers and the pooler layer.
+            num_hidden_layers: Number of hidden layers in the Transformer encoder.
+            num_attention_heads: Number of attention heads for each attention layer in
+                the Transformer encoder.
+            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
+                layer in the Transformer encoder.
+            hidden_act: The non-linear activation function (function or string) in the
+                encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob: The dropout probabilitiy for all fully connected
+                layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob: The dropout ratio for the attention
+                probabilities.
+            max_position_embeddings: The maximum sequence length that this model might
+                ever be used with. Typically set this to something large just in case
+                (e.g., 512 or 1024 or 2048).
+            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
+                `XxxModel`.
+            initializer_range: The sttdev of the truncated_normal_initializer for
+                initializing all weight matrices.
+            layer_norm_eps: The epsilon used by LayerNorm.
+    """
+    pretrained_config_archive_map = XXX_PRETRAINED_CONFIG_ARCHIVE_MAP
+
+    def __init__(self,
+                 vocab_size_or_config_json_file=50257,
+                 n_positions=1024,
+                 n_ctx=1024,
+                 n_embd=768,
+                 n_layer=12,
+                 n_head=12,
+                 resid_pdrop=0.1,
+                 embd_pdrop=0.1,
+                 attn_pdrop=0.1,
+                 layer_norm_epsilon=1e-5,
+                 initializer_range=0.02,
+
+                 num_labels=1,
+                 summary_type='cls_index',
+                 summary_use_proj=True,
+                 summary_activation=None,
+                 summary_proj_to_labels=True,
+                 summary_first_dropout=0.1,
+                 **kwargs):
+        super(XxxConfig, self).__init__(**kwargs)
+        self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, six.string_types) else -1
+        self.n_ctx = n_ctx
+        self.n_positions = n_positions
+        self.n_embd = n_embd
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attn_pdrop = attn_pdrop
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+
+        self.num_labels = num_labels
+        self.summary_type = summary_type
+        self.summary_use_proj = summary_use_proj
+        self.summary_activation = summary_activation
+        self.summary_first_dropout = summary_first_dropout
+        self.summary_proj_to_labels = summary_proj_to_labels
+        if isinstance(vocab_size_or_config_json_file, six.string_types):
+            with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader:
+                json_config = json.loads(reader.read())
+            for key, value in json_config.items():
+                self.__dict__[key] = value
+        elif not isinstance(vocab_size_or_config_json_file, int):
+            raise ValueError(
+                "First argument must be either a vocabulary size (int)"
+                "or the path to a pretrained model config file (str)"
+            )
+
+    @property
+    def max_position_embeddings(self):
+        return self.n_positions
+
+    @property
+    def hidden_size(self):
+        return self.n_embd
+
+    @property
+    def num_attention_heads(self):
+        return self.n_head
+
+    @property
+    def num_hidden_layers(self):
+        return self.n_layer
diff --git a/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py b/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py
new file mode 100755
index 0000000000..d50d129cba
--- /dev/null
+++ b/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py
@@ -0,0 +1,65 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convert XXX checkpoint."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import torch
+
+from transformers import XxxConfig, XxxForPreTraining, load_tf_weights_in_xxx
+
+import logging
+logging.basicConfig(level=logging.INFO)
+
+def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, xxx_config_file, pytorch_dump_path):
+    # Initialise PyTorch model
+    config = XxxConfig.from_json_file(xxx_config_file)
+    print("Building PyTorch model from configuration: {}".format(str(config)))
+    model = XxxForPreTraining(config)
+
+    # Load weights from tf checkpoint
+    load_tf_weights_in_xxx(model, config, tf_checkpoint_path)
+
+    # Save pytorch-model
+    print("Save PyTorch model to {}".format(pytorch_dump_path))
+    torch.save(model.state_dict(), pytorch_dump_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    ## Required parameters
+    parser.add_argument("--tf_checkpoint_path",
+                        default = None,
+                        type = str,
+                        required = True,
+                        help = "Path to the TensorFlow checkpoint path.")
+    parser.add_argument("--xxx_config_file",
+                        default = None,
+                        type = str,
+                        required = True,
+                        help = "The config json file corresponding to the pre-trained XXX model. \n"
+                            "This specifies the model architecture.")
+    parser.add_argument("--pytorch_dump_path",
+                        default = None,
+                        type = str,
+                        required = True,
+                        help = "Path to the output PyTorch model.")
+    args = parser.parse_args()
+    convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path,
+                                     args.xxx_config_file,
+                                     args.pytorch_dump_path)
diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py
new file mode 100644
index 0000000000..c661975768
--- /dev/null
+++ b/templates/adding_a_new_model/modeling_tf_xxx.py
@@ -0,0 +1,500 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" TF 2.0 XXX model. """
+
+####################################################
+# In this template, replace all the XXX (various casings) with your model name
+####################################################
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import logging
+import math
+import os
+import sys
+from io import open
+
+import numpy as np
+import tensorflow as tf
+
+from .configuration_xxx import XxxConfig
+from .modeling_tf_utils import TFPreTrainedModel, get_initializer
+from .file_utils import add_start_docstrings
+
+logger = logging.getLogger(__name__)
+
+####################################################
+# This dict contrains shortcut names and associated url
+# for the pretrained weights provided with the models
+####################################################
+TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP = {
+    'xxx-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-base-uncased-tf_model.h5",
+    'xxx-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-large-uncased-tf_model.h5",
+}
+
+####################################################
+# TF 2.0 Models are constructed using Keras imperative API by sub-classing
+# - tf.keras.layers.Layer for the layers and
+# - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model)
+####################################################
+
+####################################################
+# Here is an example of typical layer in a TF 2.0 model of the library
+# The classes are usually identical to the PyTorch ones and prefixed with 'TF'.
+#
+# Note that class __init__ parameters includes **kwargs (send to 'super').
+# This let us have a control on class scope and variable names:
+# More precisely, we set the names of the class attributes (lower level layers) to
+# to the equivalent attributes names in the PyTorch model so we can have equivalent
+# class and scope structure between PyTorch and TF 2.0 models and easily load one in the other.
+#
+# See the conversion methods in modeling_tf_pytorch_utils.py for more details
+####################################################
+class TFXxxLayer(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFXxxLayer, self).__init__(**kwargs)
+        self.attention = TFXxxAttention(config, name='attention')
+        self.intermediate = TFXxxIntermediate(config, name='intermediate')
+        self.transformer_output = TFXxxOutput(config, name='output')
+
+    def call(self, inputs, training=False):
+        hidden_states, attention_mask, head_mask = inputs
+
+        attention_outputs = self.attention([hidden_states, attention_mask, head_mask], training=training)
+        attention_output = attention_outputs[0]
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.transformer_output([intermediate_output, attention_output], training=training)
+        outputs = (layer_output,) + attention_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+####################################################
+# The full model without a specific pretrained or finetuning head is
+# provided as a tf.keras.layers.Layer usually called "TFXxxMainLayer"
+####################################################
+class TFXxxMainLayer(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFXxxMainLayer, self).__init__(**kwargs)
+
+    def _resize_token_embeddings(self, new_num_tokens):
+        raise NotImplementedError  # Not implemented yet in the library fr TF 2.0 models
+
+    def _prune_heads(self, heads_to_prune):
+        raise NotImplementedError  # Not implemented yet in the library fr TF 2.0 models
+
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+        # We allow three types of multi-inputs:
+        # - traditional keyword arguments in the call method
+        # - all the arguments provided as a dict in the first positional argument of call
+        # - all the arguments provided as a list/tuple (ordered) in the first positional argument of call
+        # The last two options are useful to use the tf.keras fit() method.
+
+        if isinstance(inputs, (tuple, list)):
+            input_ids = inputs[0]
+            attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
+            token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
+            position_ids = inputs[3] if len(inputs) > 3 else position_ids
+            head_mask = inputs[4] if len(inputs) > 4 else head_mask
+            assert len(inputs) <= 5, "Too many inputs."
+        elif isinstance(inputs, dict):
+            input_ids = inputs.get('input_ids')
+            attention_mask = inputs.get('attention_mask', attention_mask)
+            token_type_ids = inputs.get('token_type_ids', token_type_ids)
+            position_ids = inputs.get('position_ids', position_ids)
+            head_mask = inputs.get('head_mask', head_mask)
+            assert len(inputs) <= 5, "Too many inputs."
+        else:
+            input_ids = inputs
+
+        if attention_mask is None:
+            attention_mask = tf.fill(tf.shape(input_ids), 1)
+        if token_type_ids is None:
+            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+
+        # We create a 3D attention mask from a 2D tensor mask.
+        # Sizes are [batch_size, 1, 1, to_seq_length]
+        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+        # this attention mask is more simple than the triangular masking of causal attention
+        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+        extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :]
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+
+        extended_attention_mask = tf.cast(extended_attention_mask, tf.float32)
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        if not head_mask is None:
+            raise NotImplementedError
+        else:
+            head_mask = [None] * self.num_hidden_layers
+            # head_mask = tf.constant([0] * self.num_hidden_layers)
+
+        ##################################
+        # Replace this with your model code
+        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training)
+        sequence_output = encoder_outputs[0]
+        outputs = (sequence_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+
+        return outputs  # sequence_output, (hidden_states), (attentions)
+
+
+####################################################
+# TFXxxPreTrainedModel is a sub-class of tf.keras.Model
+# which take care of loading and saving pretrained weights
+# and various common utilities.
+# Here you just need to specify a few (self-explanatory)
+# pointers for your model.
+####################################################
+class TFXxxPreTrainedModel(TFPreTrainedModel):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+    config_class = XxxConfig
+    pretrained_model_archive_map = TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "transformer"
+
+
+XXX_START_DOCSTRING = r"""    The XXX model was proposed in
+    `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_
+    by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer
+    pre-trained using a combination of masked language modeling objective and next sentence prediction
+    on a large corpus comprising the Toronto Book Corpus and Wikipedia.
+
+    This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and
+    refer to the TF 2.0 documentation for all matter related to general usage and behavior.
+
+    .. _`XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`:
+        https://arxiv.org/abs/1810.04805
+
+    .. _`tf.keras.Model`:
+        https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model
+
+    Note on the model inputs:
+        TF 2.0 models accepts two formats as inputs:
+
+            - having all inputs as keyword arguments (like PyTorch models), or
+            - having all inputs as a list, tuple or dict in the first positional arguments.
+
+        This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`.
+
+        If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument :
+
+        - a single Tensor with input_ids only and nothing else: `model(inputs_ids)
+        - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
+            `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])`
+        - a dictionary with one or several input Tensors associaed to the input names given in the docstring:
+            `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})`
+
+    Parameters:
+        config (:class:`~transformers.XxxConfig`): Model configuration class with all the parameters of the model. 
+            Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
+"""
+
+XXX_INPUTS_DOCSTRING = r"""
+    Inputs:
+        **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of input sequence tokens in the vocabulary.
+            To match pre-training, XXX input sequence should be formatted with [CLS] and [SEP] tokens as follows:
+
+            (a) For sequence pairs:
+
+                ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]``
+                
+                ``token_type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1``
+
+            (b) For single sequences:
+
+                ``tokens:         [CLS] the dog is hairy . [SEP]``
+                
+                ``token_type_ids:   0   0   0   0  0     0   0``
+
+            Xxx is a model with absolute position embeddings so it's usually advised to pad the inputs on
+            the right rather than the left.
+
+            Indices can be obtained using :class:`transformers.XxxTokenizer`.
+            See :func:`transformers.PreTrainedTokenizer.encode` and
+            :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
+        **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
+        **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Segment token indices to indicate first and second portions of the inputs.
+            Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1``
+            corresponds to a `sentence B` token
+            (see `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details).
+        **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of positions of each input sequence tokens in the position embeddings.
+            Selected in the range ``[0, config.max_position_embeddings - 1]``.
+        **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:
+            Mask to nullify selected heads of the self-attention modules.
+            Mask values selected in ``[0, 1]``:
+            ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+"""
+
+@add_start_docstrings("The bare Xxx Model transformer outputing raw hidden-states without any specific head on top.",
+                      XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class TFXxxModel(TFXxxPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)``
+            Sequence of hidden-states at the output of the last layer of the model.
+        **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)``
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during Xxx pretraining. This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import XxxTokenizer, TFXxxModel
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = TFXxxModel.from_pretrained('xxx-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFXxxModel, self).__init__(config, *inputs, **kwargs)
+        self.transformer = TFXxxMainLayer(config, name='transformer')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.transformer(inputs, **kwargs)
+        return outputs
+
+
+@add_start_docstrings("""Xxx Model with a `language modeling` head on top. """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class TFXxxForMaskedLM(TFXxxPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import XxxTokenizer, TFXxxForMaskedLM
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = TFXxxForMaskedLM.from_pretrained('xxx-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        prediction_scores = outputs[0]
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFXxxForMaskedLM, self).__init__(config, *inputs, **kwargs)
+
+        self.transformer = TFXxxMainLayer(config, name='transformer')
+        self.mlm = TFXxxMLMHead(config, self.transformer.embeddings, name='mlm')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.transformer(inputs, **kwargs)
+
+        sequence_output = outputs[0]
+        prediction_scores = self.mlm(sequence_output, training=kwargs.get('training', False))
+
+        outputs = (prediction_scores,) + outputs[2:]  # Add hidden states and attention if they are here
+
+        return outputs  # prediction_scores, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model transformer with a sequence classification/regression head on top (a linear layer on top of
+    the pooled output) e.g. for GLUE tasks. """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class TFXxxForSequenceClassification(TFXxxPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import XxxTokenizer, TFXxxForSequenceClassification
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = TFXxxForSequenceClassification.from_pretrained('xxx-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        logits = outputs[0]
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFXxxForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        self.num_labels = config.num_labels
+
+        self.transformer = TFXxxMainLayer(config, name='transformer')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+        self.classifier = tf.keras.layers.Dense(config.num_labels,
+                                                kernel_initializer=get_initializer(config.initializer_range),
+                                                name='classifier')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.transformer(inputs, **kwargs)
+
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False))
+        logits = self.classifier(pooled_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        return outputs  # logits, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model with a token classification head on top (a linear layer on top of
+    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class TFXxxForTokenClassification(TFXxxPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)``
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import XxxTokenizer, TFXxxForTokenClassification
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = TFXxxForTokenClassification.from_pretrained('xxx-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        scores = outputs[0]
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFXxxForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        self.num_labels = config.num_labels
+
+        self.transformer = TFXxxMainLayer(config, name='transformer')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+        self.classifier = tf.keras.layers.Dense(config.num_labels,
+                                                kernel_initializer=get_initializer(config.initializer_range),
+                                                name='classifier')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.transformer(inputs, **kwargs)
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False))
+        logits = self.classifier(sequence_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        return outputs  # scores, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of
+    the hidden-states output to compute `span start logits` and `span end logits`). """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class TFXxxForQuestionAnswering(TFXxxPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **start_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)``
+            Span-start scores (before SoftMax).
+        **end_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)``
+            Span-end scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import XxxTokenizer, TFXxxForQuestionAnswering
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = TFXxxForQuestionAnswering.from_pretrained('xxx-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        start_scores, end_scores = outputs[:2]
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFXxxForQuestionAnswering, self).__init__(config, *inputs, **kwargs)
+        self.num_labels = config.num_labels
+
+        self.transformer = TFXxxMainLayer(config, name='transformer')
+        self.qa_outputs = tf.keras.layers.Dense(config.num_labels,
+                                                kernel_initializer=get_initializer(config.initializer_range),
+                                                name='qa_outputs')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.transformer(inputs, **kwargs)
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = tf.split(logits, 2, axis=-1)
+        start_logits = tf.squeeze(start_logits, axis=-1)
+        end_logits = tf.squeeze(end_logits, axis=-1)
+
+        outputs = (start_logits, end_logits,) + outputs[2:]
+
+        return outputs  # start_logits, end_logits, (hidden_states), (attentions)
diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
new file mode 100644
index 0000000000..7e2ba9dfb5
--- /dev/null
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -0,0 +1,644 @@
+# coding=utf-8
+# Copyright 2018 XXX Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" PyTorch XXX model. """
+
+####################################################
+# In this template, replace all the XXX (various casings) with your model name
+####################################################
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import logging
+import math
+import os
+import sys
+from io import open
+
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss, MSELoss
+
+from .modeling_utils import PreTrainedModel, prune_linear_layer
+from .configuration_xxx import XxxConfig
+from .file_utils import add_start_docstrings
+
+logger = logging.getLogger(__name__)
+
+####################################################
+# This dict contrains shortcut names and associated url
+# for the pretrained weights provided with the models
+####################################################
+XXX_PRETRAINED_MODEL_ARCHIVE_MAP = {
+    'xxx-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-base-uncased-pytorch_model.bin",
+    'xxx-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-large-uncased-pytorch_model.bin",
+}
+
+####################################################
+# This is a conversion method from TF 1.0 to PyTorch
+# More details: https://medium.com/huggingface/from-tensorflow-to-pytorch-265f40ef2a28
+####################################################
+def load_tf_weights_in_xxx(model, config, tf_checkpoint_path):
+    """ Load tf checkpoints in a pytorch model.
+    """
+    try:
+        import re
+        import numpy as np
+        import tensorflow as tf
+    except ImportError:
+        logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
+            "https://www.tensorflow.org/install/ for installation instructions.")
+        raise
+    tf_path = os.path.abspath(tf_checkpoint_path)
+    logger.info("Converting TensorFlow checkpoint from {}".format(tf_path))
+    # Load weights from TF model
+    init_vars = tf.train.list_variables(tf_path)
+    names = []
+    arrays = []
+    for name, shape in init_vars:
+        logger.info("Loading TF weight {} with shape {}".format(name, shape))
+        array = tf.train.load_variable(tf_path, name)
+        names.append(name)
+        arrays.append(array)
+
+    for name, array in zip(names, arrays):
+        name = name.split('/')
+        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
+        # which are not required for using pretrained model
+        if any(n in ["adam_v", "adam_m", "global_step"] for n in name):
+            logger.info("Skipping {}".format("/".join(name)))
+            continue
+        pointer = model
+        for m_name in name:
+            if re.fullmatch(r'[A-Za-z]+_\d+', m_name):
+                l = re.split(r'_(\d+)', m_name)
+            else:
+                l = [m_name]
+            if l[0] == 'kernel' or l[0] == 'gamma':
+                pointer = getattr(pointer, 'weight')
+            elif l[0] == 'output_bias' or l[0] == 'beta':
+                pointer = getattr(pointer, 'bias')
+            elif l[0] == 'output_weights':
+                pointer = getattr(pointer, 'weight')
+            elif l[0] == 'squad':
+                pointer = getattr(pointer, 'classifier')
+            else:
+                try:
+                    pointer = getattr(pointer, l[0])
+                except AttributeError:
+                    logger.info("Skipping {}".format("/".join(name)))
+                    continue
+            if len(l) >= 2:
+                num = int(l[1])
+                pointer = pointer[num]
+        if m_name[-11:] == '_embeddings':
+            pointer = getattr(pointer, 'weight')
+        elif m_name == 'kernel':
+            array = np.transpose(array)
+        try:
+            assert pointer.shape == array.shape
+        except AssertionError as e:
+            e.args += (pointer.shape, array.shape)
+            raise
+        logger.info("Initialize PyTorch weight {}".format(name))
+        pointer.data = torch.from_numpy(array)
+    return model
+
+
+####################################################
+# PyTorch Models are constructed by sub-classing
+# - torch.nn.Module for the layers and
+# - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module)
+####################################################
+
+####################################################
+# Here is an example of typical layer in a PyTorch model of the library
+# The classes are usually identical to the TF 2.0 ones without the 'TF' prefix.
+#
+# See the conversion methods in modeling_tf_pytorch_utils.py for more details
+####################################################
+class XxxLayer(nn.Module):
+    def __init__(self, config):
+        super(XxxLayer, self).__init__()
+        self.attention = XxxAttention(config)
+        self.intermediate = XxxIntermediate(config)
+        self.output = XxxOutput(config)
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
+        attention_output = attention_outputs[0]
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        outputs = (layer_output,) + attention_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+
+####################################################
+# PreTrainedModel is a sub-class of torch.nn.Module
+# which take care of loading and saving pretrained weights
+# and various common utilities.
+#
+# Here you just need to specify a few (self-explanatory)
+# pointers for your model and the weights initialization
+# method if its not fully covered by PreTrainedModel's default method
+####################################################
+class XxxPreTrainedModel(PreTrainedModel):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+    config_class = XxxConfig
+    pretrained_model_archive_map = XXX_PRETRAINED_MODEL_ARCHIVE_MAP
+    load_tf_weights = load_tf_weights_in_xxx
+    base_model_prefix = "transformer"
+
+    def _init_weights(self, module):
+        """ Initialize the weights """
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, XxxLayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            module.bias.data.zero_()
+
+
+XXX_START_DOCSTRING = r"""    The XXX model was proposed in
+    `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_
+    by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer
+    pre-trained using a combination of masked language modeling objective and next sentence prediction
+    on a large corpus comprising the Toronto Book Corpus and Wikipedia.
+
+    This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and
+    refer to the PyTorch documentation for all matter related to general usage and behavior.
+
+    .. _`XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`:
+        https://arxiv.org/abs/1810.04805
+
+    .. _`torch.nn.Module`:
+        https://pytorch.org/docs/stable/nn.html#module
+
+    Parameters:
+        config (:class:`~transformers.XxxConfig`): Model configuration class with all the parameters of the model. 
+            Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
+"""
+
+XXX_INPUTS_DOCSTRING = r"""
+    Inputs:
+        **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of input sequence tokens in the vocabulary.
+            To match pre-training, XXX input sequence should be formatted with [CLS] and [SEP] tokens as follows:
+
+            (a) For sequence pairs:
+
+                ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]``
+                
+                ``token_type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1``
+
+            (b) For single sequences:
+
+                ``tokens:         [CLS] the dog is hairy . [SEP]``
+                
+                ``token_type_ids:   0   0   0   0  0     0   0``
+
+            Xxx is a model with absolute position embeddings so it's usually advised to pad the inputs on
+            the right rather than the left.
+
+            Indices can be obtained using :class:`transformers.XxxTokenizer`.
+            See :func:`transformers.PreTrainedTokenizer.encode` and
+            :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
+        **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:
+            Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
+        **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Segment token indices to indicate first and second portions of the inputs.
+            Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1``
+            corresponds to a `sentence B` token
+            (see `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details).
+        **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of positions of each input sequence tokens in the position embeddings.
+            Selected in the range ``[0, config.max_position_embeddings - 1]``.
+        **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:
+            Mask to nullify selected heads of the self-attention modules.
+            Mask values selected in ``[0, 1]``:
+            ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+"""
+
+@add_start_docstrings("The bare Xxx Model transformer outputting raw hidden-states without any specific head on top.",
+                      XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class XxxModel(XxxPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
+            Sequence of hidden-states at the output of the last layer of the model.
+        **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)``
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during Xxx pretraining. This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = XxxModel.from_pretrained('xxx-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
+    """
+    def __init__(self, config):
+        super(XxxModel, self).__init__(config)
+
+        self.embeddings = XxxEmbeddings(config)
+        self.encoder = XxxEncoder(config)
+        self.pooler = XxxPooler(config)
+
+        self.init_weights()
+
+    def _resize_token_embeddings(self, new_num_tokens):
+        old_embeddings = self.embeddings.word_embeddings
+        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
+        self.embeddings.word_embeddings = new_embeddings
+        return self.embeddings.word_embeddings
+
+    def _prune_heads(self, heads_to_prune):
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            See base class PreTrainedModel
+        """
+        for layer, heads in heads_to_prune.items():
+            self.encoder.layer[layer].attention.prune_heads(heads)
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros_like(input_ids)
+
+        # We create a 3D attention mask from a 2D tensor mask.
+        # Sizes are [batch_size, 1, 1, to_seq_length]
+        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+        # this attention mask is more simple than the triangular masking of causal attention
+        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        if head_mask is not None:
+            if head_mask.dim() == 1:
+                head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
+                head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1)
+            elif head_mask.dim() == 2:
+                head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
+            head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility
+        else:
+            head_mask = [None] * self.config.num_hidden_layers
+
+        ##################################
+        # Replace this with your model code
+        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask)
+        sequence_output = encoder_outputs[0]
+        outputs = (sequence_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+
+        return outputs  # sequence_output, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model with a `language modeling` head on top. """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class XxxForMaskedLM(XxxPreTrainedModel):
+    r"""
+        **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the masked language modeling loss.
+            Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
+            Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
+            in ``[0, ..., config.vocab_size]``
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Masked language modeling loss.
+        **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = XxxForMaskedLM.from_pretrained('xxx-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, masked_lm_labels=input_ids)
+        loss, prediction_scores = outputs[:2]
+
+    """
+    def __init__(self, config):
+        super(XxxForMaskedLM, self).__init__(config)
+
+        self.transformer = XxxModel(config)
+        self.cls = XxxOnlyMLMHead(config)
+
+        self.init_weights()
+        self.tie_weights()
+
+    def tie_weights(self):
+        """ Make sure we are sharing the input and output embeddings.
+            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
+        """
+        self._tie_or_clone_weights(self.cls.predictions.decoder,
+                                   self.transformer.embeddings.word_embeddings)
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                masked_lm_labels=None):
+
+        outputs = self.transformer(input_ids,
+                            attention_mask=attention_mask,
+                            token_type_ids=token_type_ids,
+                            position_ids=position_ids, 
+                            head_mask=head_mask)
+
+        sequence_output = outputs[0]
+        prediction_scores = self.cls(sequence_output)
+
+        outputs = (prediction_scores,) + outputs[2:]  # Add hidden states and attention if they are here
+        if masked_lm_labels is not None:
+            loss_fct = CrossEntropyLoss(ignore_index=-1)
+            masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
+            outputs = (masked_lm_loss,) + outputs
+
+        return outputs  # (masked_lm_loss), prediction_scores, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model transformer with a sequence classification/regression head on top (a linear layer on top of
+    the pooled output) e.g. for GLUE tasks. """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class XxxForSequenceClassification(XxxPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = XxxForSequenceClassification.from_pretrained('xxx-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+
+    """
+    def __init__(self, config):
+        super(XxxForSequenceClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.transformer = XxxModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, labels=None):
+
+        outputs = self.transformer(input_ids,
+                            attention_mask=attention_mask,
+                            token_type_ids=token_type_ids,
+                            position_ids=position_ids, 
+                            head_mask=head_mask)
+
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        if labels is not None:
+            if self.num_labels == 1:
+                #  We are doing regression
+                loss_fct = MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model with a token classification head on top (a linear layer on top of
+    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class XxxForTokenClassification(XxxPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the token classification loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification loss.
+        **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)``
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = XxxForTokenClassification.from_pretrained('xxx-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, scores = outputs[:2]
+
+    """
+    def __init__(self, config):
+        super(XxxForTokenClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.transformer = XxxModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, labels=None):
+
+        outputs = self.transformer(input_ids,
+                            attention_mask=attention_mask,
+                            token_type_ids=token_type_ids,
+                            position_ids=position_ids, 
+                            head_mask=head_mask)
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output)
+        logits = self.classifier(sequence_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+            # Only keep active parts of the loss
+            if attention_mask is not None:
+                active_loss = attention_mask.view(-1) == 1
+                active_logits = logits.view(-1, self.num_labels)[active_loss]
+                active_labels = labels.view(-1)[active_loss]
+                loss = loss_fct(active_logits, active_labels)
+            else:
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), scores, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Xxx Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of
+    the hidden-states output to compute `span start logits` and `span end logits`). """,
+    XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING)
+class XxxForQuestionAnswering(XxxPreTrainedModel):
+    r"""
+        **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for position (index) of the start of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`).
+            Position outside of the sequence are not taken into account for computing the loss.
+        **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for position (index) of the end of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`).
+            Position outside of the sequence are not taken into account for computing the loss.
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
+        **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
+            Span-start scores (before SoftMax).
+        **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
+            Span-end scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased')
+        model = XxxForQuestionAnswering.from_pretrained('xxx-large-uncased-whole-word-masking-finetuned-squad')
+        question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+        input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]"
+        input_ids = tokenizer.encode(input_text)
+        token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] 
+        start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))
+        all_tokens = tokenizer.convert_ids_to_tokens(input_ids)  
+        print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1]))
+        # a nice puppet
+
+
+    """
+    def __init__(self, config):
+        super(XxxForQuestionAnswering, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.transformer = XxxModel(config)
+        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                start_positions=None, end_positions=None):
+
+        outputs = self.transformer(input_ids,
+                            attention_mask=attention_mask,
+                            token_type_ids=token_type_ids,
+                            position_ids=position_ids, 
+                            head_mask=head_mask)
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1)
+        end_logits = end_logits.squeeze(-1)
+
+        outputs = (start_logits, end_logits,) + outputs[2:]
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions.clamp_(0, ignored_index)
+            end_positions.clamp_(0, ignored_index)
+
+            loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+            outputs = (total_loss,) + outputs
+
+        return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)
diff --git a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py
new file mode 100644
index 0000000000..90837ca1ea
--- /dev/null
+++ b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py
@@ -0,0 +1,256 @@
+# coding=utf-8
+# Copyright 2018 XXX Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+import shutil
+import pytest
+import sys
+
+from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor)
+from .configuration_common_test import ConfigTester
+
+from transformers import XxxConfig, is_tf_available
+
+if is_tf_available():
+    import tensorflow as tf
+    from transformers.modeling_tf_xxx import (TFXxxModel, TFXxxForMaskedLM,
+                                               TFXxxForSequenceClassification,
+                                               TFXxxForTokenClassification,
+                                               TFXxxForQuestionAnswering,
+                                               TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP)
+else:
+    pytestmark = pytest.mark.skip("Require TensorFlow")
+
+
+class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester):
+
+    all_model_classes = (TFXxxModel, TFXxxForMaskedLM, TFXxxForQuestionAnswering,
+                         TFXxxForSequenceClassification,
+                         TFXxxForTokenClassification) if is_tf_available() else ()
+
+    class TFXxxModelTester(object):
+
+        def __init__(self,
+                     parent,
+                     batch_size=13,
+                     seq_length=7,
+                     is_training=True,
+                     use_input_mask=True,
+                     use_token_type_ids=True,
+                     use_labels=True,
+                     vocab_size=99,
+                     hidden_size=32,
+                     num_hidden_layers=5,
+                     num_attention_heads=4,
+                     intermediate_size=37,
+                     hidden_act="gelu",
+                     hidden_dropout_prob=0.1,
+                     attention_probs_dropout_prob=0.1,
+                     max_position_embeddings=512,
+                     type_vocab_size=16,
+                     type_sequence_label_size=2,
+                     initializer_range=0.02,
+                     num_labels=3,
+                     num_choices=4,
+                     scope=None,
+                    ):
+            self.parent = parent
+            self.batch_size = batch_size
+            self.seq_length = seq_length
+            self.is_training = is_training
+            self.use_input_mask = use_input_mask
+            self.use_token_type_ids = use_token_type_ids
+            self.use_labels = use_labels
+            self.vocab_size = vocab_size
+            self.hidden_size = hidden_size
+            self.num_hidden_layers = num_hidden_layers
+            self.num_attention_heads = num_attention_heads
+            self.intermediate_size = intermediate_size
+            self.hidden_act = hidden_act
+            self.hidden_dropout_prob = hidden_dropout_prob
+            self.attention_probs_dropout_prob = attention_probs_dropout_prob
+            self.max_position_embeddings = max_position_embeddings
+            self.type_vocab_size = type_vocab_size
+            self.type_sequence_label_size = type_sequence_label_size
+            self.initializer_range = initializer_range
+            self.num_labels = num_labels
+            self.num_choices = num_choices
+            self.scope = scope
+
+        def prepare_config_and_inputs(self):
+            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
+
+            input_mask = None
+            if self.use_input_mask:
+                input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
+
+            token_type_ids = None
+            if self.use_token_type_ids:
+                token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
+
+            sequence_labels = None
+            token_labels = None
+            choice_labels = None
+            if self.use_labels:
+                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
+                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
+                choice_labels = ids_tensor([self.batch_size], self.num_choices)
+
+            config = XxxConfig(
+                vocab_size_or_config_json_file=self.vocab_size,
+                hidden_size=self.hidden_size,
+                num_hidden_layers=self.num_hidden_layers,
+                num_attention_heads=self.num_attention_heads,
+                intermediate_size=self.intermediate_size,
+                hidden_act=self.hidden_act,
+                hidden_dropout_prob=self.hidden_dropout_prob,
+                attention_probs_dropout_prob=self.attention_probs_dropout_prob,
+                max_position_embeddings=self.max_position_embeddings,
+                type_vocab_size=self.type_vocab_size,
+                initializer_range=self.initializer_range)
+
+            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+
+        def create_and_check_xxx_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = TFXxxModel(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            sequence_output, pooled_output = model(inputs)
+
+            inputs = [input_ids, input_mask]
+            sequence_output, pooled_output = model(inputs)
+
+            sequence_output, pooled_output = model(input_ids)
+
+            result = {
+                "sequence_output": sequence_output.numpy(),
+                "pooled_output": pooled_output.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["sequence_output"].shape),
+                [self.batch_size, self.seq_length, self.hidden_size])
+            self.parent.assertListEqual(list(result["pooled_output"].shape), [self.batch_size, self.hidden_size])
+
+
+        def create_and_check_xxx_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = TFXxxForMaskedLM(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            prediction_scores, = model(inputs)
+            result = {
+                "prediction_scores": prediction_scores.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["prediction_scores"].shape),
+                [self.batch_size, self.seq_length, self.vocab_size])
+
+
+        def create_and_check_xxx_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = TFXxxForSequenceClassification(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            logits, = model(inputs)
+            result = {
+                "logits": logits.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].shape),
+                [self.batch_size, self.num_labels])
+
+
+        def create_and_check_xxx_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = TFXxxForTokenClassification(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            logits, = model(inputs)
+            result = {
+                "logits": logits.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].shape),
+                [self.batch_size, self.seq_length, self.num_labels])
+
+
+        def create_and_check_xxx_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = TFXxxForQuestionAnswering(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            start_logits, end_logits = model(inputs)
+            result = {
+                "start_logits": start_logits.numpy(),
+                "end_logits": end_logits.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["start_logits"].shape),
+                [self.batch_size, self.seq_length])
+            self.parent.assertListEqual(
+                list(result["end_logits"].shape),
+                [self.batch_size, self.seq_length])
+
+
+        def prepare_config_and_inputs_for_common(self):
+            config_and_inputs = self.prepare_config_and_inputs()
+            (config, input_ids, token_type_ids, input_mask,
+             sequence_labels, token_labels, choice_labels) = config_and_inputs
+            inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
+            return config, inputs_dict
+
+    def setUp(self):
+        self.model_tester = TFXxxModelTest.TFXxxModelTester(self)
+        self.config_tester = ConfigTester(self, config_class=XxxConfig, hidden_size=37)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_xxx_model(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_model(*config_and_inputs)
+
+    def test_for_masked_lm(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_masked_lm(*config_and_inputs)
+
+    def test_for_question_answering(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_question_answering(*config_and_inputs)
+
+    def test_for_sequence_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_sequence_classification(*config_and_inputs)
+
+    def test_for_token_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs)
+
+    @pytest.mark.slow
+    def test_model_from_pretrained(self):
+        cache_dir = "/tmp/transformers_test/"
+        for model_name in ['xxx-base-uncased']:
+            model = TFXxxModel.from_pretrained(model_name, cache_dir=cache_dir)
+            shutil.rmtree(cache_dir)
+            self.assertIsNotNone(model)
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/templates/adding_a_new_model/tests/modeling_xxx_test.py b/templates/adding_a_new_model/tests/modeling_xxx_test.py
new file mode 100644
index 0000000000..8c0cc3cf32
--- /dev/null
+++ b/templates/adding_a_new_model/tests/modeling_xxx_test.py
@@ -0,0 +1,255 @@
+# coding=utf-8
+# Copyright 2018 XXX Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+import shutil
+import pytest
+
+from transformers import is_torch_available
+
+from .modeling_common_test import (CommonTestCases, ids_tensor)
+from .configuration_common_test import ConfigTester
+
+if is_torch_available():
+    from transformers import (XxxConfig, XxxModel, XxxForMaskedLM,
+                                        XxxForNextSentencePrediction, XxxForPreTraining,
+                                        XxxForQuestionAnswering, XxxForSequenceClassification,
+                                        XxxForTokenClassification, XxxForMultipleChoice)
+    from transformers.modeling_xxx import XXX_PRETRAINED_MODEL_ARCHIVE_MAP
+else:
+    pytestmark = pytest.mark.skip("Require Torch")
+
+
+class XxxModelTest(CommonTestCases.CommonModelTester):
+
+    all_model_classes = (XxxModel, XxxForMaskedLM, XxxForQuestionAnswering,
+                         XxxForSequenceClassification,
+                         XxxForTokenClassification) if is_torch_available() else ()
+
+    class XxxModelTester(object):
+
+        def __init__(self,
+                     parent,
+                     batch_size=13,
+                     seq_length=7,
+                     is_training=True,
+                     use_input_mask=True,
+                     use_token_type_ids=True,
+                     use_labels=True,
+                     vocab_size=99,
+                     hidden_size=32,
+                     num_hidden_layers=5,
+                     num_attention_heads=4,
+                     intermediate_size=37,
+                     hidden_act="gelu",
+                     hidden_dropout_prob=0.1,
+                     attention_probs_dropout_prob=0.1,
+                     max_position_embeddings=512,
+                     type_vocab_size=16,
+                     type_sequence_label_size=2,
+                     initializer_range=0.02,
+                     num_labels=3,
+                     num_choices=4,
+                     scope=None,
+                    ):
+            self.parent = parent
+            self.batch_size = batch_size
+            self.seq_length = seq_length
+            self.is_training = is_training
+            self.use_input_mask = use_input_mask
+            self.use_token_type_ids = use_token_type_ids
+            self.use_labels = use_labels
+            self.vocab_size = vocab_size
+            self.hidden_size = hidden_size
+            self.num_hidden_layers = num_hidden_layers
+            self.num_attention_heads = num_attention_heads
+            self.intermediate_size = intermediate_size
+            self.hidden_act = hidden_act
+            self.hidden_dropout_prob = hidden_dropout_prob
+            self.attention_probs_dropout_prob = attention_probs_dropout_prob
+            self.max_position_embeddings = max_position_embeddings
+            self.type_vocab_size = type_vocab_size
+            self.type_sequence_label_size = type_sequence_label_size
+            self.initializer_range = initializer_range
+            self.num_labels = num_labels
+            self.num_choices = num_choices
+            self.scope = scope
+
+        def prepare_config_and_inputs(self):
+            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
+
+            input_mask = None
+            if self.use_input_mask:
+                input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
+
+            token_type_ids = None
+            if self.use_token_type_ids:
+                token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
+
+            sequence_labels = None
+            token_labels = None
+            choice_labels = None
+            if self.use_labels:
+                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
+                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
+                choice_labels = ids_tensor([self.batch_size], self.num_choices)
+
+            config = XxxConfig(
+                vocab_size_or_config_json_file=self.vocab_size,
+                hidden_size=self.hidden_size,
+                num_hidden_layers=self.num_hidden_layers,
+                num_attention_heads=self.num_attention_heads,
+                intermediate_size=self.intermediate_size,
+                hidden_act=self.hidden_act,
+                hidden_dropout_prob=self.hidden_dropout_prob,
+                attention_probs_dropout_prob=self.attention_probs_dropout_prob,
+                max_position_embeddings=self.max_position_embeddings,
+                type_vocab_size=self.type_vocab_size,
+                initializer_range=self.initializer_range)
+
+            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+
+        def check_loss_output(self, result):
+            self.parent.assertListEqual(
+                list(result["loss"].size()),
+                [])
+
+        def create_and_check_xxx_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = XxxModel(config=config)
+            model.eval()
+            sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
+            sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
+            sequence_output, pooled_output = model(input_ids)
+
+            result = {
+                "sequence_output": sequence_output,
+                "pooled_output": pooled_output,
+            }
+            self.parent.assertListEqual(
+                list(result["sequence_output"].size()),
+                [self.batch_size, self.seq_length, self.hidden_size])
+            self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size])
+
+
+        def create_and_check_xxx_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = XxxForMaskedLM(config=config)
+            model.eval()
+            loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels)
+            result = {
+                "loss": loss,
+                "prediction_scores": prediction_scores,
+            }
+            self.parent.assertListEqual(
+                list(result["prediction_scores"].size()),
+                [self.batch_size, self.seq_length, self.vocab_size])
+            self.check_loss_output(result)
+
+
+        def create_and_check_xxx_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = XxxForQuestionAnswering(config=config)
+            model.eval()
+            loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids,
+                                                   start_positions=sequence_labels, end_positions=sequence_labels)
+            result = {
+                "loss": loss,
+                "start_logits": start_logits,
+                "end_logits": end_logits,
+            }
+            self.parent.assertListEqual(
+                list(result["start_logits"].size()),
+                [self.batch_size, self.seq_length])
+            self.parent.assertListEqual(
+                list(result["end_logits"].size()),
+                [self.batch_size, self.seq_length])
+            self.check_loss_output(result)
+
+
+        def create_and_check_xxx_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = XxxForSequenceClassification(config)
+            model.eval()
+            loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.num_labels])
+            self.check_loss_output(result)
+
+
+        def create_and_check_xxx_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = XxxForTokenClassification(config=config)
+            model.eval()
+            loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.seq_length, self.num_labels])
+            self.check_loss_output(result)
+
+
+        def prepare_config_and_inputs_for_common(self):
+            config_and_inputs = self.prepare_config_and_inputs()
+            (config, input_ids, token_type_ids, input_mask,
+             sequence_labels, token_labels, choice_labels) = config_and_inputs
+            inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
+            return config, inputs_dict
+
+    def setUp(self):
+        self.model_tester = XxxModelTest.XxxModelTester(self)
+        self.config_tester = ConfigTester(self, config_class=XxxConfig, hidden_size=37)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_xxx_model(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_model(*config_and_inputs)
+
+    def test_for_masked_lm(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_masked_lm(*config_and_inputs)
+
+    def test_for_question_answering(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_question_answering(*config_and_inputs)
+
+    def test_for_sequence_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_sequence_classification(*config_and_inputs)
+
+    def test_for_token_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs)
+
+    @pytest.mark.slow
+    def test_model_from_pretrained(self):
+        cache_dir = "/tmp/transformers_test/"
+        for model_name in list(XXX_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
+            model = XxxModel.from_pretrained(model_name, cache_dir=cache_dir)
+            shutil.rmtree(cache_dir)
+            self.assertIsNotNone(model)
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/templates/adding_a_new_model/tests/tokenization_xxx_test.py b/templates/adding_a_new_model/tests/tokenization_xxx_test.py
new file mode 100644
index 0000000000..116083edc8
--- /dev/null
+++ b/templates/adding_a_new_model/tests/tokenization_xxx_test.py
@@ -0,0 +1,57 @@
+# coding=utf-8
+# Copyright 2018 XXX Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+import unittest
+from io import open
+
+from transformers.tokenization_bert import (XxxTokenizer, VOCAB_FILES_NAMES)
+
+from .tokenization_tests_commons import CommonTestCases
+
+class XxxTokenizationTest(CommonTestCases.CommonTokenizerTester):
+
+    tokenizer_class = XxxTokenizer
+
+    def setUp(self):
+        super(XxxTokenizationTest, self).setUp()
+
+        vocab_tokens = [
+            "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn",
+            "##ing", ",", "low", "lowest",
+        ]
+        self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file'])
+        with open(self.vocab_file, "w", encoding='utf-8') as vocab_writer:
+            vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
+
+    def get_tokenizer(self, **kwargs):
+        return XxxTokenizer.from_pretrained(self.tmpdirname, **kwargs)
+
+    def get_input_output_texts(self):
+        input_text = u"UNwant\u00E9d,running"
+        output_text = u"unwanted, running"
+        return input_text, output_text
+
+    def test_full_tokenizer(self):
+        tokenizer = self.tokenizer_class(self.vocab_file)
+
+        tokens = tokenizer.tokenize(u"UNwant\u00E9d,running")
+        self.assertListEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"])
+        self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/templates/adding_a_new_model/tokenization_xxx.py b/templates/adding_a_new_model/tokenization_xxx.py
new file mode 100644
index 0000000000..1b1325aab5
--- /dev/null
+++ b/templates/adding_a_new_model/tokenization_xxx.py
@@ -0,0 +1,218 @@
+# coding=utf-8
+# Copyright 2018 XXX Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization class for model XXX."""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import collections
+import logging
+import os
+import unicodedata
+from io import open
+
+from .tokenization_utils import PreTrainedTokenizer
+
+logger = logging.getLogger(__name__)
+
+####################################################
+# In this template, replace all the XXX (various casings) with your model name
+####################################################
+
+####################################################
+# Mapping from the keyword arguments names of Tokenizer `__init__`
+# to file names for serializing Tokenizer instances
+####################################################
+VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'}
+
+####################################################
+# Mapping from the keyword arguments names of Tokenizer `__init__`
+# to pretrained vocabulary URL for all the model shortcut names.
+####################################################
+PRETRAINED_VOCAB_FILES_MAP = {
+    'vocab_file':
+    {
+        'xxx-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-base-uncased-vocab.txt",
+        'xxx-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/xxx-large-uncased-vocab.txt",
+    }
+}
+
+####################################################
+# Mapping from model shortcut names to max length of inputs
+####################################################
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    'xxx-base-uncased': 512,
+    'xxx-large-uncased': 512,
+}
+
+####################################################
+# Mapping from model shortcut names to a dictionary of additional
+# keyword arguments for Tokenizer `__init__`.
+# To be used for checkpoint specific configurations.
+####################################################
+PRETRAINED_INIT_CONFIGURATION = {
+    'xxx-base-uncased': {'do_lower_case': True},
+    'xxx-large-uncased': {'do_lower_case': True},
+}
+
+
+def load_vocab(vocab_file):
+    """Loads a vocabulary file into a dictionary."""
+    vocab = collections.OrderedDict()
+    with open(vocab_file, "r", encoding="utf-8") as reader:
+        tokens = reader.readlines()
+    for index, token in enumerate(tokens):
+        token = token.rstrip('\n')
+        vocab[token] = index
+    return vocab
+
+
+class XxxTokenizer(PreTrainedTokenizer):
+    r"""
+    Constructs a XxxTokenizer.
+    :class:`~transformers.XxxTokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece
+
+    Args:
+        vocab_file: Path to a one-wordpiece-per-line vocabulary file
+        do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(self, vocab_file, do_lower_case=True,
+                 unk_token="[UNK]", sep_token="[SEP]", pad_token="[PAD]", cls_token="[CLS]",
+                 mask_token="[MASK]", **kwargs):
+        """Constructs a XxxTokenizer.
+
+        Args:
+            **vocab_file**: Path to a one-wordpiece-per-line vocabulary file
+            **do_lower_case**: (`optional`) boolean (default True)
+                Whether to lower case the input
+                Only has an effect when do_basic_tokenize=True
+        """
+        super(XxxTokenizer, self).__init__(unk_token=unk_token, sep_token=sep_token,
+                                           pad_token=pad_token, cls_token=cls_token,
+                                           mask_token=mask_token, **kwargs)
+        self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
+        self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
+
+        if not os.path.isfile(vocab_file):
+            raise ValueError(
+                "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
+                "model use `tokenizer = XxxTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file))
+        self.vocab = load_vocab(vocab_file)
+
+    @property
+    def vocab_size(self):
+        return len(self.vocab)
+
+    def _tokenize(self, text):
+        """ Take as input a string and return a list of strings (tokens) for words/sub-words
+        """
+        split_tokens = []
+        if self.do_basic_tokenize:
+            for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens):
+                for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                    split_tokens.append(sub_token)
+        else:
+            split_tokens = self.wordpiece_tokenizer.tokenize(text)
+        return split_tokens
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str/unicode) in an id using the vocab. """
+        return self.vocab.get(token, self.vocab.get(self.unk_token))
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (string/unicode) using the vocab."""
+        return self.ids_to_tokens.get(index, self.unk_token)
+
+    def convert_tokens_to_string(self, tokens):
+        """ Converts a sequence of tokens (string) in a single string. """
+        out_string = ' '.join(tokens).replace(' ##', '').strip()
+        return out_string
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        by concatenating and adding special tokens.
+        A BERT sequence has the following format:
+            single sequence: [CLS] X [SEP]
+            pair of sequences: [CLS] A [SEP] B [SEP]
+        """
+        if token_ids_1 is None:
+            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+        cls = [self.cls_token_id]
+        sep = [self.sep_token_id]
+        return cls + token_ids_0 + sep + token_ids_1 + sep
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
+
+        Args:
+            token_ids_0: list of ids (must not contain special tokens)
+            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
+                for sequence pairs
+            already_has_special_tokens: (default False) Set to True if the token list is already formated with
+                special tokens for the model
+
+        Returns:
+            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            if token_ids_1 is not None:
+                raise ValueError("You should not supply a second sequence if the provided sequence of "
+                                 "ids is already formated with special tokens for the model.")
+            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
+
+        if token_ids_1 is not None:
+            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
+        A BERT sequence pair mask has the following format:
+        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence
+
+        if token_ids_1 is None, only returns the first portion of the mask (0's).
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+        if token_ids_1 is None:
+            return len(cls + token_ids_0 + sep) * [0]
+        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]
+
+    def save_vocabulary(self, vocab_path):
+        """Save the tokenizer vocabulary to a directory or file."""
+        index = 0
+        if os.path.isdir(vocab_path):
+            vocab_file = os.path.join(vocab_path, VOCAB_FILES_NAMES['vocab_file'])
+        else:
+            vocab_file = vocab_path
+        with open(vocab_file, "w", encoding="utf-8") as writer:
+            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning("Saving vocabulary to {}: vocabulary indices are not consecutive."
+                                   " Please check that the vocabulary is not corrupted!".format(vocab_file))
+                    index = token_index
+                writer.write(token + u'\n')
+                index += 1
+        return (vocab_file,)

From 328a86d2af6b2c4d4b57cbed9f0a6eb0b3ad6256 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Wed, 30 Oct 2019 11:37:55 +0100
Subject: [PATCH 107/269] adding links to the templates in readme and
 contributing

---
 CONTRIBUTING.md | 4 ++++
 README.md       | 1 +
 2 files changed, 5 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 817ba56aaf..69f6af78ca 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -62,6 +62,8 @@ Awesome! Please provide the following information:
 If you are willing to contribute the model yourself, let us know so we can best
 guide you.
 
+We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find then in the [`templates`](./templates) folder.
+
 ### Do you want a new feature (that is not a model)?
 
 A world-class feature request addresses the following points:
@@ -81,6 +83,8 @@ A world-class feature request addresses the following points:
 If your issue is well written we're already 80% of the way there by the time you
 post it.
 
+We have added **templates** to guide you in the process of adding a new example script for training or testing the models in the library. You can find then in the [`templates`](./templates) folder.
+
 ## Start contributing! (Pull Requests)
 
 Before writing code, we strongly advise you to search through the exising PRs or
diff --git a/README.md b/README.md
index ecba50a74e..12167e6e9c 100644
--- a/README.md
+++ b/README.md
@@ -122,6 +122,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation).
 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
+10. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
 

From cef2a8f9002dbee18ede7762a3595722468c6de3 Mon Sep 17 00:00:00 2001
From: Thomas Wolf <thomwolf@users.noreply.github.com>
Date: Wed, 30 Oct 2019 12:25:31 +0100
Subject: [PATCH 108/269] Update CONTRIBUTING.md

Co-Authored-By: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 69f6af78ca..db70cc644e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -62,7 +62,7 @@ Awesome! Please provide the following information:
 If you are willing to contribute the model yourself, let us know so we can best
 guide you.
 
-We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find then in the [`templates`](./templates) folder.
+We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder.
 
 ### Do you want a new feature (that is not a model)?
 

From 55fbfea369d9b43cbceeb362806ef4ca56b88acc Mon Sep 17 00:00:00 2001
From: Thomas Wolf <thomwolf@users.noreply.github.com>
Date: Wed, 30 Oct 2019 12:25:40 +0100
Subject: [PATCH 109/269] Update CONTRIBUTING.md

Co-Authored-By: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index db70cc644e..136ef8df81 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -83,7 +83,7 @@ A world-class feature request addresses the following points:
 If your issue is well written we're already 80% of the way there by the time you
 post it.
 
-We have added **templates** to guide you in the process of adding a new example script for training or testing the models in the library. You can find then in the [`templates`](./templates) folder.
+We have added **templates** to guide you in the process of adding a new example script for training or testing the models in the library. You can find them in the [`templates`](./templates) folder.
 
 ## Start contributing! (Pull Requests)
 

From 3f07cd419ce973b4ca8fd4e12fe664d08408b343 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 15:09:53 +0100
Subject: [PATCH 110/269] update test on Bert to include decoder mode

---
 transformers/tests/modeling_bert_test.py | 50 +++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index 6c39c4e4db..67be910a7e 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -22,7 +22,7 @@ import pytest
 
 from transformers import is_torch_available
 
-from .modeling_common_test import (CommonTestCases, ids_tensor)
+from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor)
 from .configuration_common_test import ConfigTester
 
 if is_torch_available():
@@ -120,10 +120,20 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 attention_probs_dropout_prob=self.attention_probs_dropout_prob,
                 max_position_embeddings=self.max_position_embeddings,
                 type_vocab_size=self.type_vocab_size,
+                is_decoder=False,
                 initializer_range=self.initializer_range)
 
             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 
+        def prepare_config_and_inputs_for_decoder(self):
+            config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels = self.prepare_config_and_inputs()
+
+            config.is_decoder = True
+            encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
+            encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
+
+            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask
+
         def check_loss_output(self, result):
             self.parent.assertListEqual(
                 list(result["loss"].size()),
@@ -145,6 +155,22 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.hidden_size])
             self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size])
 
+        def create_and_check_bert_model_as_decoder(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask):
+            model = BertModel(config)
+            model.eval()
+            sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask)
+            sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, encoder_hidden_states=encoder_hidden_states)
+            sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
+
+            result = {
+                "sequence_output": sequence_output,
+                "pooled_output": pooled_output,
+            }
+            self.parent.assertListEqual(
+                list(result["sequence_output"].size()),
+                [self.batch_size, self.seq_length, self.hidden_size])
+            self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size])
+
         def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = BertForMaskedLM(config=config)
             model.eval()
@@ -158,6 +184,20 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.vocab_size])
             self.check_loss_output(result)
 
+        def create_and_check_bert_model_for_masked_lm_as_decoder(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask):
+            model = BertForMaskedLM(config=config)
+            model.eval()
+            loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask)
+            loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, encoder_hidden_states=encoder_hidden_states)
+            result = {
+                "loss": loss,
+                "prediction_scores": prediction_scores,
+            }
+            self.parent.assertListEqual(
+                list(result["prediction_scores"].size()),
+                [self.batch_size, self.seq_length, self.vocab_size])
+            self.check_loss_output(result)
+
         def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = BertForNextSentencePrediction(config=config)
             model.eval()
@@ -273,10 +313,18 @@ class BertModelTest(CommonTestCases.CommonModelTester):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_bert_model(*config_and_inputs)
 
+    def test_bert_model_as_decoder(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
+        self.model_tester.create_and_check_bert_model_as_decoder(*config_and_inputs)
+
     def test_for_masked_lm(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs)
 
+    def test_for_masked_lm_decoder(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
+        self.model_tester.create_and_check_bert_model_for_masked_lm_as_decoder(*config_and_inputs)
+
     def test_for_multiple_choice(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs)

From a88a0e4413d8de5ad235a211fb3b0326aadc5ce0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 16:06:29 +0100
Subject: [PATCH 111/269] add tests to encoder-decoder model

---
 transformers/tests/modeling_common_test.py    | 16 ++++++
 .../tests/modeling_encoder_decoder_test.py    | 52 +++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 transformers/tests/modeling_encoder_decoder_test.py

diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 2b66757c28..1c1794550c 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -704,6 +704,22 @@ def ids_tensor(shape, vocab_size, rng=None, name=None):
     return torch.tensor(data=values, dtype=torch.long).view(shape).contiguous()
 
 
+def floats_tensor(shape, scale=1.0, rng=None, name=None):
+    """Creates a random float32 tensor of the shape within the vocab size."""
+    if rng is None:
+        rng = global_rng
+
+    total_dims = 1
+    for dim in shape:
+        total_dims *= dim
+
+    values = []
+    for _ in range(total_dims):
+        values.append(rng.random() * scale)
+
+    return torch.tensor(data=values, dtype=torch.float).view(shape).contiguous()
+
+
 class ModelUtilsTest(unittest.TestCase):
     def test_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
diff --git a/transformers/tests/modeling_encoder_decoder_test.py b/transformers/tests/modeling_encoder_decoder_test.py
new file mode 100644
index 0000000000..1ffd0ebc4c
--- /dev/null
+++ b/transformers/tests/modeling_encoder_decoder_test.py
@@ -0,0 +1,52 @@
+# coding=utf-8
+# Copyright 2018 The Hugging Face Inc. Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import unittest
+import pytest
+
+from transformers import is_torch_available
+
+if is_torch_available():
+    from transformers import BertModel, BertForMaskedLM, Model2Model
+    from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP
+else:
+    pytestmark = pytest.mark.skip("Require Torch")
+
+
+class EncoderDecoderModelTest(unittest.TestCase):
+    def test_model2model_from_pretrained(self):
+        logging.basicConfig(level=logging.INFO)
+        for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
+            model = Model2Model.from_pretrained(model_name)
+            self.assertIsInstance(model.encoder, BertModel)
+            self.assertIsInstance(model.decoder, BertForMaskedLM)
+            self.assertEqual(model.decoder.config.is_decoder, True)
+            self.assertEqual(model.encoder.config.is_decoder, False)
+
+    def test_model2model_from_pretrained_not_bert(self):
+        logging.basicConfig(level=logging.INFO)
+        with self.assertRaises(ValueError):
+            _ = Model2Model.from_pretrained('roberta')
+
+        with self.assertRaises(ValueError):
+            _ = Model2Model.from_pretrained('distilbert')
+
+        with self.assertRaises(ValueError):
+            _ = Model2Model.from_pretrained('does-not-exist')
+
+
+if __name__ == "__main__":
+    unittest.main()

From 3cf2020c6becb3fb9c8f3c6db684184b5cdb2ac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 30 Oct 2019 16:27:51 +0100
Subject: [PATCH 112/269] change kwargs processing

---
 transformers/modeling_encoder_decoder.py | 71 ++++++++++++++----------
 1 file changed, 43 insertions(+), 28 deletions(-)

diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py
index 162e2f8b3b..a884abd0a2 100644
--- a/transformers/modeling_encoder_decoder.py
+++ b/transformers/modeling_encoder_decoder.py
@@ -114,23 +114,28 @@ class PreTrainedEncoderDecoder(nn.Module):
         # `encoder_`), decoder-specific (prefixed by `decoder_`) and those
         # that apply to the model as a whole.
         # We let the specific kwargs override the common ones in case of conflict.
-        kwargs_encoder = {
-            argument[len("encoder_"):]: value
-            for argument, value in kwargs.items()
-            if argument.startswith("encoder_")
-        }
-        kwargs_decoder = {
-            argument[len("decoder_"):]: value
-            for argument, value in kwargs.items()
-            if argument.startswith("decoder_")
-        }
         kwargs_common = {
             argument: value
             for argument, value in kwargs.items()
-            if not (argument.startswith("encoder_") or argument.startswith("decoder_"))
+            if not argument.startswith("encoder_")
+            and not argument.startswith("decoder_")
         }
-        kwargs_decoder = dict(kwargs_common, **kwargs_decoder)
-        kwargs_encoder = dict(kwargs_common, **kwargs_encoder)
+        kwargs_decoder = kwargs_common.copy()
+        kwargs_encoder = kwargs_common.copy()
+        kwargs_encoder.update(
+            {
+                argument[len("encoder_") :]: value
+                for argument, value in kwargs.items()
+                if argument.startswith("encoder_")
+            }
+        )
+        kwargs_decoder.update(
+            {
+                argument[len("decoder_") :]: value
+                for argument, value in kwargs.items()
+                if argument.startswith("decoder_")
+            }
+        )
 
         # Load and initialize the encoder and decoder
         # The distinction between encoder and decoder at the model level is made
@@ -185,35 +190,44 @@ class PreTrainedEncoderDecoder(nn.Module):
         # `encoder_`), decoder-specific (prefixed by `decoder_`) and those
         # that apply to the model as whole.
         # We let the specific kwargs override the common ones in case of conflict.
-        kwargs_encoder = {
-            argument[len("encoder_"):]: value
-            for argument, value in kwargs.items()
-            if argument.startswith("encoder_")
-        }
-        kwargs_decoder = {
-            argument[len("decoder_"):]: value
-            for argument, value in kwargs.items()
-            if argument.startswith("decoder_")
-        }
         kwargs_common = {
             argument: value
             for argument, value in kwargs.items()
-            if not (argument.startswith("encoder_") or argument.startswith("decoder_"))
+            if not argument.startswith("encoder_")
+            and not argument.startswith("decoder_")
         }
-        kwargs_decoder = dict(kwargs_common, **kwargs_decoder)
-        kwargs_encoder = dict(kwargs_common, **kwargs_encoder)
+        kwargs_decoder = kwargs_common.copy()
+        kwargs_encoder = kwargs_common.copy()
+        kwargs_encoder.update(
+            {
+                argument[len("encoder_") :]: value
+                for argument, value in kwargs.items()
+                if argument.startswith("encoder_")
+            }
+        )
+        kwargs_decoder.update(
+            {
+                argument[len("decoder_") :]: value
+                for argument, value in kwargs.items()
+                if argument.startswith("decoder_")
+            }
+        )
 
         # Encode if needed (training, first prediction pass)
         encoder_hidden_states = kwargs_encoder.pop("hidden_states", None)
         if encoder_hidden_states is None:
             encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder)
-            encoder_hidden_states = encoder_outputs[0]  # output the last layer hidden state
+            encoder_hidden_states = encoder_outputs[
+                0
+            ]  # output the last layer hidden state
         else:
             encoder_outputs = ()
 
         # Decode
         kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states
-        kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None)
+        kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get(
+            "attention_mask", None
+        )
         decoder_outputs = self.decoder(decoder_input_ids, **kwargs_decoder)
 
         return decoder_outputs + encoder_outputs
@@ -235,6 +249,7 @@ class Model2Model(PreTrainedEncoderDecoder):
             decoder = BertForMaskedLM(config)
             model = Model2Model.from_pretrained('bert-base-uncased', decoder_model=decoder)
     """
+
     def __init__(self, *args, **kwargs):
         super(Model2Model, self).__init__(*args, **kwargs)
         self.tie_weights()

From fa735208c96c18283b8d2f3fcbfc3157bbd12b1e Mon Sep 17 00:00:00 2001
From: Victor SANH <victorsanh@gmail.com>
Date: Wed, 30 Oct 2019 14:27:28 -0400
Subject: [PATCH 113/269] update readme - fix example command distil*

---
 examples/distillation/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/distillation/README.md b/examples/distillation/README.md
index 7da1ad015b..8efd1ea6f4 100644
--- a/examples/distillation/README.md
+++ b/examples/distillation/README.md
@@ -108,7 +108,7 @@ python train.py \
     --student_config training_configs/distilbert-base-uncased.json \
     --teacher_type bert \
     --teacher_name bert-base-uncased \
-    --alpha_ce 5.0 --alpha_mlm 2.0 --alpha_cos 1.0 --mlm \
+    --alpha_ce 5.0 --alpha_mlm 2.0 --alpha_cos 1.0 --alpha_clm 0.0 --mlm \
     --freeze_pos_embs \
     --dump_path serialization_dir/my_first_training \
     --data_file data/binarized_text.bert-base-uncased.pickle \
@@ -144,7 +144,7 @@ python -m torch.distributed.launch \
         --student_config training_configs/distilbert-base-uncased.json \
         --teacher_type bert \
         --teacher_name bert-base-uncased \
-        --alpha_ce 0.33 --alpha_mlm 0.33 --alpha_cos 0.33 --mlm \
+        --alpha_ce 0.33 --alpha_mlm 0.33 --alpha_cos 0.33 --alpha_clm 0.0 --mlm \
         --freeze_pos_embs \
         --dump_path serialization_dir/my_first_training \
         --data_file data/binarized_text.bert-base-uncased.pickle \
@@ -166,4 +166,4 @@ If you find the ressource useful, you should cite the following paper:
   booktitle={NeurIPS EMC^2 Workshop},
   year={2019}
 }
-```
\ No newline at end of file
+```

From ac29353abe4ba30a488c14cbfb897ed41dc0fa05 Mon Sep 17 00:00:00 2001
From: cregouby <cregouby@users.noreply.github.com>
Date: Thu, 31 Oct 2019 10:04:40 +0100
Subject: [PATCH 114/269] Fix
 https://github.com/huggingface/transformers/issues/1673

---
 transformers/modeling_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index c88cab143a..03490630ed 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -284,6 +284,7 @@ class PreTrainedModel(nn.Module):
                 pretrained_model_name_or_path, *model_args,
                 cache_dir=cache_dir, return_unused_kwargs=True,
                 force_download=force_download,
+                proxies=proxies,
                 **kwargs
             )
         else:

From f96ce1c24151349251880c95e9a9fb144b62367c Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Thu, 31 Oct 2019 18:27:11 +0000
Subject: [PATCH 115/269] [run_generation] Fix generation with batch_size>1

---
 examples/run_generation.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/examples/run_generation.py b/examples/run_generation.py
index b8cc8a9bbf..2d917660cf 100644
--- a/examples/run_generation.py
+++ b/examples/run_generation.py
@@ -79,7 +79,7 @@ def set_seed(args):
 def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')):
     """ Filter a distribution of logits using top-k and/or nucleus (top-p) filtering
         Args:
-            logits: logits distribution shape (vocabulary size)
+            logits: logits distribution shape (batch size x vocabulary size)
             top_k > 0: keep only top k tokens with highest probability (top-k filtering).
             top_p > 0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).
                 Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)
@@ -138,13 +138,14 @@ def sample_sequence(model, length, context, num_samples=1, temperature=1, top_k=
             outputs = model(**inputs)  # Note: we could also use 'past' with GPT-2/Transfo-XL/XLNet/CTRL (cached hidden-states)
             next_token_logits = outputs[0][:, -1, :] / (temperature if temperature > 0 else 1.)
 
-            # reptition penalty from CTRL (https://arxiv.org/abs/1909.05858)
-            for _ in set(generated.view(-1).tolist()):
-                next_token_logits[_] /= repetition_penalty
+            # repetition penalty from CTRL (https://arxiv.org/abs/1909.05858)
+            for i in range(num_samples):
+                for _ in set(generated[i].tolist()):
+                    next_token_logits[i, _] /= repetition_penalty
                 
             filtered_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)
-            if temperature == 0: #greedy sampling:
-                next_token = torch.argmax(filtered_logits).unsqueeze(0)
+            if temperature == 0: # greedy sampling:
+                next_token = torch.argmax(filtered_logits, dim=-1).unsqueeze(-1)
             else:
                 next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1), num_samples=1)
             generated = torch.cat((generated, next_token), dim=1)

From be36cf92fb96de1766b092c282c7609dcbe4a8e7 Mon Sep 17 00:00:00 2001
From: Timothy Liu <tlkh.xms@gmail.com>
Date: Wed, 30 Oct 2019 00:44:23 +0000
Subject: [PATCH 116/269] Added mixed precision support to benchmarks.py

---
 examples/benchmarks.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/examples/benchmarks.py b/examples/benchmarks.py
index 06f368d946..26c260b9ec 100644
--- a/examples/benchmarks.py
+++ b/examples/benchmarks.py
@@ -253,18 +253,22 @@ def create_setup_and_compute(model_names: List[str],
                              average_over: int = 3,
                              torchscript: bool = False,
                              xla: bool = False,
+                             amp: bool = False,
+                             fp16: bool = False,
                              save_to_csv: bool = False,
                              csv_filename: str = f"results_{round(time())}.csv"):
     if xla:
         tf.config.optimizer.set_jit(True)
+    if amp:
+        tf.config.optimizer.set_experimental_options({"auto_mixed_precision": True})
 
     if tensorflow:
         dictionary = {model_name: {} for model_name in model_names}
-        results = _compute_tensorflow(model_names, dictionary, average_over)
+        results = _compute_tensorflow(model_names, dictionary, average_over, amp)
     else:
         device = 'cuda' if (gpu and torch.cuda.is_available()) else 'cpu'
         dictionary = {model_name: {} for model_name in model_names}
-        results = _compute_pytorch(model_names, dictionary, average_over, device, torchscript)
+        results = _compute_pytorch(model_names, dictionary, average_over, device, torchscript, fp16)
 
     print("=========== RESULTS ===========")
     for model_name in model_names:
@@ -302,7 +306,7 @@ def create_setup_and_compute(model_names: List[str],
                 writer.writerow({'model': model_name, **model_results})
 
 
-def _compute_pytorch(model_names, dictionary, average_over, device, torchscript):
+def _compute_pytorch(model_names, dictionary, average_over, device, torchscript, fp16):
     for c, model_name in enumerate(model_names):
         print(f"{c + 1} / {len(model_names)}")
         config = AutoConfig.from_pretrained(model_name, torchscript=torchscript)
@@ -319,6 +323,8 @@ def _compute_pytorch(model_names, dictionary, average_over, device, torchscript)
         dictionary[model_name]["results"] = {i: {} for i in batch_sizes}
 
         for batch_size in batch_sizes:
+            if fp16:
+                model.half()
             model.to(device)
             model.eval()
             for slice_size in slice_sizes:
@@ -346,7 +352,7 @@ def _compute_pytorch(model_names, dictionary, average_over, device, torchscript)
     return dictionary
 
 
-def _compute_tensorflow(model_names, dictionary, average_over):
+def _compute_tensorflow(model_names, dictionary, average_over, amp):
     for c, model_name in enumerate(model_names):
         print(f"{c + 1} / {len(model_names)}")
         config = AutoConfig.from_pretrained(model_name)
@@ -409,6 +415,8 @@ def main():
                                                                                   "the correct dependencies are "
                                                                                   "installed")
     parser.add_argument("--xla", required=False, action="store_true", help="TensorFlow only: use XLA acceleration.")
+    parser.add_argument("--amp", required=False, action="store_true", help="TensorFlow only: use automatic mixed precision acceleration.")
+    parser.add_argument("--fp16", required=False, action="store_true", help="PyTorch only: use FP16 to accelerate inference.")
     parser.add_argument("--keras_predict", required=False, action="store_true", help="Whether to use model.predict "
                                                                                      "instead of model() to do a "
                                                                                      "forward pass.")
@@ -442,6 +450,7 @@ def main():
                 tensorflow=False,
                 gpu=args.torch_cuda,
                 torchscript=args.torchscript,
+                fp16=args.fp16,
                 save_to_csv=args.save_to_csv,
                 csv_filename=args.csv_filename,
                 average_over=args.average_over
@@ -455,6 +464,7 @@ def main():
                 model_names=args.models,
                 tensorflow=True,
                 xla=args.xla,
+                amp=args.amp,
                 save_to_csv=args.save_to_csv,
                 csv_filename=args.csv_filename,
                 average_over=args.average_over

From 1a2b40cb53477b94c66718bac8d997297fcc8043 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 31 Oct 2019 18:00:51 -0400
Subject: [PATCH 117/269] run_tf_glue MRPC evaluation only for MRPC

---
 examples/run_tf_glue.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py
index 73173b0cf1..8878ce726e 100644
--- a/examples/run_tf_glue.py
+++ b/examples/run_tf_glue.py
@@ -71,20 +71,21 @@ history = model.fit(train_dataset, epochs=EPOCHS, steps_per_epoch=train_steps,
 os.makedirs('./save/', exist_ok=True)
 model.save_pretrained('./save/')
 
-# Load the TensorFlow model in PyTorch for inspection
-pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True)
+if TASK == "mrpc":
+    # Load the TensorFlow model in PyTorch for inspection
+    pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True)
 
-# Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task
-sentence_0 = 'This research was consistent with his findings.'
-sentence_1 = 'His findings were compatible with this research.'
-sentence_2 = 'His findings were not compatible with this research.'
-inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')
-inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt')
+    # Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task
+    sentence_0 = 'This research was consistent with his findings.'
+    sentence_1 = 'His findings were compatible with this research.'
+    sentence_2 = 'His findings were not compatible with this research.'
+    inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')
+    inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt')
 
-del inputs_1["special_tokens_mask"]
-del inputs_2["special_tokens_mask"]
+    del inputs_1["special_tokens_mask"]
+    del inputs_2["special_tokens_mask"]
 
-pred_1 = pytorch_model(**inputs_1)[0].argmax().item()
-pred_2 = pytorch_model(**inputs_2)[0].argmax().item()
-print('sentence_1 is', 'a paraphrase' if pred_1 else 'not a paraphrase', 'of sentence_0')
-print('sentence_2 is', 'a paraphrase' if pred_2 else 'not a paraphrase', 'of sentence_0')
+    pred_1 = pytorch_model(**inputs_1)[0].argmax().item()
+    pred_2 = pytorch_model(**inputs_2)[0].argmax().item()
+    print('sentence_1 is', 'a paraphrase' if pred_1 else 'not a paraphrase', 'of sentence_0')
+    print('sentence_2 is', 'a paraphrase' if pred_2 else 'not a paraphrase', 'of sentence_0')

From 93d2fff0716d83df168ca0686d16bc4cd7ccb366 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Fri, 1 Nov 2019 09:47:38 -0400
Subject: [PATCH 118/269] Close #1654

---
 docs/source/model_doc/ctrl.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/source/model_doc/ctrl.rst b/docs/source/model_doc/ctrl.rst
index 9fd5b4acdb..36b37b3ee1 100644
--- a/docs/source/model_doc/ctrl.rst
+++ b/docs/source/model_doc/ctrl.rst
@@ -1,6 +1,11 @@
 CTRL
 ----------------------------------------------------
 
+Note: if you fine-tune a CTRL model using the Salesforce code (https://github.com/salesforce/ctrl),
+you'll be able to convert from TF to our HuggingFace/Transformers format using the 
+``convert_tf_to_huggingface_pytorch.py`` script (see `issue #1654 <https://github.com/huggingface/transformers/issues/1654>`_).
+
+
 ``CTRLConfig``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From e5b1048bae5b0992fad6a342ed168e4327e54b2a Mon Sep 17 00:00:00 2001
From: Raghavan <oneraghavan@gmail.com>
Date: Sun, 3 Nov 2019 16:14:46 +0530
Subject: [PATCH 119/269] Fixing mode in evaluate during training

---
 examples/run_ner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index aeb6a9a292..3649b84d34 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -153,7 +153,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id):
                 if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
                     # Log metrics
                     if args.local_rank == -1 and args.evaluate_during_training:  # Only evaluate when single GPU otherwise metrics may not average well
-                        results, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id)
+                        results, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode="dev")
                         for key, value in results.items():
                             tb_writer.add_scalar("eval_{}".format(key), value, global_step)
                     tb_writer.add_scalar("lr", scheduler.get_lr()[0], global_step)

From ad908686272801d9c53effad8ec513ef2cda85dd Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 4 Nov 2019 11:27:22 +0100
Subject: [PATCH 120/269] Update example readme

---
 examples/README.md | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/examples/README.md b/examples/README.md
index 382d794fcb..9f00c45225 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -8,7 +8,7 @@ similar API between the different models.
 | [Language Model fine-tuning](#language-model-fine-tuning) | Fine-tuning the library models for language modeling on a text dataset. Causal language modeling for GPT/GPT-2, masked language modeling for BERT/RoBERTa. |
 | [Language Generation](#language-generation) | Conditional text generation using the auto-regressive models of the library: GPT, GPT-2, Transformer-XL and XLNet.                                         |
 | [GLUE](#glue) | Examples running BERT/XLM/XLNet/RoBERTa on the 9 GLUE tasks. Examples feature distributed training as well as half-precision.                              |
-| [SQuAD](#squad) | Using BERT for question answering, examples with distributed training.                                                                                  |
+| [SQuAD](#squad) | Using BERT/RoBERTa/XLNet/XLM for question answering, examples with distributed training.                                                                                  |
 | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. 
 
 ## Language model fine-tuning
@@ -390,3 +390,40 @@ exact_match = 86.91
 This fine-tuneds model is available as a checkpoint under the reference
 `bert-large-uncased-whole-word-masking-finetuned-squad`.
 
+#### Fine-tuning XLNet on SQuAD
+
+This example code fine-tunes XLNet on the SQuAD dataset. See above to download the data for SQuAD .
+
+```bash
+export SQUAD_DIR=/path/to/SQUAD
+
+python /data/home/hlu/transformers/examples/run_squad.py \
+    --model_type xlnet \
+    --model_name_or_path xlnet-large-cased \
+    --do_train \
+    --do_eval \
+    --do_lower_case \
+    --train_file /data/home/hlu/notebooks/NLP/examples/question_answering/train-v1.1.json \
+    --predict_file /data/home/hlu/notebooks/NLP/examples/question_answering/dev-v1.1.json \
+    --learning_rate 3e-5 \
+    --num_train_epochs 2 \
+    --max_seq_length 384 \
+    --doc_stride 128 \
+    --output_dir ./wwm_cased_finetuned_squad/ \
+    --per_gpu_eval_batch_size=4  \
+    --per_gpu_train_batch_size=4   \
+    --save_steps 5000
+```
+
+Training with the previously defined hyper-parameters yields the following results:
+
+```python
+{
+"exact": 85.45884578997162,
+"f1": 92.5974600601065,
+"total": 10570,
+"HasAns_exact": 85.45884578997162,
+"HasAns_f1": 92.59746006010651,
+"HasAns_total": 10570
+}
+```

From 9b45d0f8787a19570a04732b0875c94951870766 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 4 Nov 2019 12:28:56 +0100
Subject: [PATCH 121/269] Add common properties input_embeddings and
 output_embeddings

---
 templates/adding_a_new_model/modeling_xxx.py |  22 ++--
 transformers/modeling_bert.py                |  30 ++---
 transformers/modeling_ctrl.py                |  17 +--
 transformers/modeling_distilbert.py          |  23 ++--
 transformers/modeling_gpt2.py                |  28 ++---
 transformers/modeling_openai.py              |  28 ++---
 transformers/modeling_roberta.py             |  13 +-
 transformers/modeling_transfo_xl.py          |   8 +-
 transformers/modeling_utils.py               | 122 ++++++++++++-------
 transformers/modeling_xlm.py                 |  16 +--
 transformers/modeling_xlnet.py               |  16 +--
 transformers/tests/modeling_common_test.py   |   9 ++
 12 files changed, 179 insertions(+), 153 deletions(-)

diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
index 7e2ba9dfb5..5335439dfb 100644
--- a/templates/adding_a_new_model/modeling_xxx.py
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -280,12 +280,14 @@ class XxxModel(XxxPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        old_embeddings = self.embeddings.word_embeddings
-        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
-        self.embeddings.word_embeddings = new_embeddings
+    @property
+    def input_embeddings(self):
         return self.embeddings.word_embeddings
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.embeddings.word_embeddings = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -376,17 +378,13 @@ class XxxForMaskedLM(XxxPreTrainedModel):
         super(XxxForMaskedLM, self).__init__(config)
 
         self.transformer = XxxModel(config)
-        self.cls = XxxOnlyMLMHead(config)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.cls.predictions.decoder,
-                                   self.transformer.embeddings.word_embeddings)
+    @property
+    def output_embeddings(self):
+        return self.lm_head
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None):
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index ff13a45bad..a920aa86d3 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -601,12 +601,14 @@ class BertModel(BertPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        old_embeddings = self.embeddings.word_embeddings
-        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
-        self.embeddings.word_embeddings = new_embeddings
+    @property
+    def input_embeddings(self):
         return self.embeddings.word_embeddings
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.embeddings.word_embeddings = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -750,14 +752,10 @@ class BertForPreTraining(BertPreTrainedModel):
         self.cls = BertPreTrainingHeads(config)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.cls.predictions.decoder,
-                                   self.bert.embeddings.word_embeddings)
+    @property
+    def output_embeddings(self):
+        return self.cls.predictions.decoder
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None, next_sentence_label=None):
@@ -830,14 +828,10 @@ class BertForMaskedLM(BertPreTrainedModel):
         self.cls = BertOnlyMLMHead(config)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.cls.predictions.decoder,
-                                   self.bert.embeddings.word_embeddings)
+    @property
+    def output_embeddings(self):
+        return self.cls.predictions.decoder
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, lm_labels=None, ):
diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py
index 55e64d318b..c588dc30ba 100644
--- a/transformers/modeling_ctrl.py
+++ b/transformers/modeling_ctrl.py
@@ -289,10 +289,14 @@ class CTRLModel(CTRLPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        self.w = self._get_resized_embeddings(self.w, new_num_tokens)
+    @property
+    def input_embeddings(self):
         return self.w
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.w = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
                 heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -449,13 +453,10 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=True)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-                Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.lm_head, self.transformer.w)
+    @property
+    def output_embeddings(self):
+        return self.lm_head
 
     def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 labels=None):
diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py
index d3b4ccff5d..7365d1a7dc 100644
--- a/transformers/modeling_distilbert.py
+++ b/transformers/modeling_distilbert.py
@@ -334,9 +334,6 @@ class DistilBertPreTrainedModel(PreTrainedModel):
     load_tf_weights = None
     base_model_prefix = "distilbert"
 
-    def __init__(self, *inputs, **kwargs):
-        super(DistilBertPreTrainedModel, self).__init__(*inputs, **kwargs)
-    
     def _init_weights(self, module):
         """ Initialize the weights.
         """
@@ -424,12 +421,14 @@ class DistilBertModel(DistilBertPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        old_embeddings = self.embeddings.word_embeddings
-        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
-        self.embeddings.word_embeddings = new_embeddings
+    @property
+    def input_embeddings(self):
         return self.embeddings.word_embeddings
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.embeddings.word_embeddings = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -511,16 +510,12 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel):
         self.vocab_projector = nn.Linear(config.dim, config.vocab_size)
 
         self.init_weights()
-        self.tie_weights()
 
         self.mlm_loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.vocab_projector,
-                                   self.distilbert.embeddings.word_embeddings)
+    @property
+    def output_embeddings(self):
+        return self.vocab_projector
 
     def forward(self, input_ids, attention_mask=None, head_mask=None, masked_lm_labels=None):
         dlbrt_output = self.distilbert(input_ids=input_ids,
diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index 0b5b83aa75..9f48152884 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -357,10 +357,14 @@ class GPT2Model(GPT2PreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        self.wte = self._get_resized_embeddings(self.wte, new_num_tokens)
+    @property
+    def input_embeddings(self):
         return self.wte
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -514,14 +518,10 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.lm_head,
-                                   self.transformer.wte)
+    @property
+    def output_embeddings(self):
+        return self.lm_head
 
     def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 labels=None):
@@ -622,14 +622,10 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
         self.multiple_choice_head = SequenceSummary(config)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.lm_head,
-                                   self.transformer.wte)
+    @property
+    def output_embeddings(self):
+        return self.lm_head
 
     def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 mc_token_ids=None, lm_labels=None, mc_labels=None):
diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py
index 52f3b7db72..9e25b9cfb4 100644
--- a/transformers/modeling_openai.py
+++ b/transformers/modeling_openai.py
@@ -360,10 +360,14 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        self.tokens_embed = self._get_resized_embeddings(self.tokens_embed, new_num_tokens)
+    @property
+    def input_embeddings(self):
         return self.tokens_embed
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.tokens_embed = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -489,14 +493,10 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel):
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.lm_head,
-                                   self.transformer.tokens_embed)
+    @property
+    def output_embeddings(self):
+        return self.lm_head
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 labels=None):
@@ -583,14 +583,10 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
         self.multiple_choice_head = SequenceSummary(config)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.lm_head,
-                                   self.transformer.tokens_embed)
+    @property
+    def output_embeddings(self):
+        return self.lm_head
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 mc_token_ids=None, lm_labels=None, mc_labels=None):
diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index c155856be7..81216c93d4 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -169,6 +169,10 @@ class RobertaModel(BertModel):
         self.embeddings = RobertaEmbeddings(config)
         self.init_weights()
 
+    @property
+    def input_embeddings(self):
+        return self.embeddings.word_embeddings
+
 
 @add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """,
     ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
@@ -213,13 +217,10 @@ class RobertaForMaskedLM(BertPreTrainedModel):
         self.lm_head = RobertaLMHead(config)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the input and output embeddings.
-            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
-        """
-        self._tie_or_clone_weights(self.lm_head.decoder, self.roberta.embeddings.word_embeddings)
+    @property
+    def output_embeddings(self):
+        return self.lm_head.decoder
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None):
diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py
index 6d430e1804..0bc7fadd77 100644
--- a/transformers/modeling_transfo_xl.py
+++ b/transformers/modeling_transfo_xl.py
@@ -639,9 +639,14 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
+    @property
+    def input_embeddings(self):
         return self.word_emb
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.word_emb = new_embeddings
+
     def backward_compatible(self):
         self.sample_softmax = -1
 
@@ -826,7 +831,6 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
             self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model,
                                                     config.cutoffs, div_val=config.div_val)
         self.init_weights()
-        self.tie_weights()
 
     def tie_weights(self):
         """
diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index 03490630ed..3c2d8e199d 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -83,6 +83,77 @@ class PreTrainedModel(nn.Module):
         # Save config in model
         self.config = config
 
+    @property
+    def base_model(self):
+        return getattr(self, self.base_model_prefix, self)
+
+    @property
+    def input_embeddings(self):
+        base_model = getattr(self, self.base_model_prefix, self)
+        return base_model.input_embeddings
+
+    @property
+    def output_embeddings(self):
+        return None  # Overwrite for models with output embeddings
+
+    def tie_weights(self):
+        """ Make sure we are sharing the input and output embeddings.
+            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
+        """
+        if self.output_embeddings is not None:
+            self._tie_or_clone_weights(self.output_embeddings, self.input_embeddings)
+
+    def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
+        """ Tie or clone module weights depending of weither we are using TorchScript or not
+        """
+        if self.config.torchscript:
+            output_embeddings.weight = nn.Parameter(input_embeddings.weight.clone())
+        else:
+            output_embeddings.weight = input_embeddings.weight
+
+        if hasattr(output_embeddings, 'bias') and output_embeddings.bias is not None:
+            output_embeddings.bias.data = torch.nn.functional.pad(
+                output_embeddings.bias.data,
+                (0, output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0]),
+                'constant',
+                0
+            )
+        if hasattr(output_embeddings, 'out_features') and hasattr(input_embeddings, 'num_embeddings'):
+            output_embeddings.out_features = input_embeddings.num_embeddings
+
+    def resize_token_embeddings(self, new_num_tokens=None):
+        """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size.
+        Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
+
+        Arguments:
+
+            new_num_tokens: (`optional`) int:
+                New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end.
+                If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model.
+
+        Return: ``torch.nn.Embeddings``
+            Pointer to the input tokens Embeddings Module of the model
+        """
+        base_model = getattr(self, self.base_model_prefix, self)  # get the base model if needed
+        model_embeds = base_model._resize_token_embeddings(new_num_tokens)
+        if new_num_tokens is None:
+            return model_embeds
+
+        # Update base model and current model config
+        self.config.vocab_size = new_num_tokens
+        base_model.vocab_size = new_num_tokens
+
+        # Tie weights again if needed
+        if hasattr(self, 'tie_weights'):
+            self.tie_weights()
+
+        return model_embeds
+
+    def _resize_token_embeddings(self, new_num_tokens):
+        old_embeddings = self.input_embeddings
+        self.input_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
+        return self.input_embeddings
+
     def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None):
         """ Build a resized Embedding Module from a provided token Embedding Module.
             Increasing the size will add newly initialized vectors at the end
@@ -117,50 +188,6 @@ class PreTrainedModel(nn.Module):
 
         return new_embeddings
 
-    def _tie_or_clone_weights(self, first_module, second_module):
-        """ Tie or clone module weights depending of weither we are using TorchScript or not
-        """
-        if self.config.torchscript:
-            first_module.weight = nn.Parameter(second_module.weight.clone())
-        else:
-            first_module.weight = second_module.weight
-
-        if hasattr(first_module, 'bias') and first_module.bias is not None:
-            first_module.bias.data = torch.nn.functional.pad(
-                first_module.bias.data,
-                (0, first_module.weight.shape[0] - first_module.bias.shape[0]),
-                'constant',
-                0
-            )
-
-    def resize_token_embeddings(self, new_num_tokens=None):
-        """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size.
-        Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
-
-        Arguments:
-
-            new_num_tokens: (`optional`) int:
-                New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end.
-                If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model.
-
-        Return: ``torch.nn.Embeddings``
-            Pointer to the input tokens Embeddings Module of the model
-        """
-        base_model = getattr(self, self.base_model_prefix, self)  # get the base model if needed
-        model_embeds = base_model._resize_token_embeddings(new_num_tokens)
-        if new_num_tokens is None:
-            return model_embeds
-
-        # Update base model and current model config
-        self.config.vocab_size = new_num_tokens
-        base_model.vocab_size = new_num_tokens
-
-        # Tie weights again if needed
-        if hasattr(self, 'tie_weights'):
-            self.tie_weights()
-
-        return model_embeds
-
     def init_weights(self):
         """ Initialize and prunes weights if needed. """
         # Initialize weights
@@ -170,6 +197,9 @@ class PreTrainedModel(nn.Module):
         if self.config.pruned_heads:
             self.prune_heads(self.config.pruned_heads)
 
+        # Tie weights if needed
+        self.tie_weights()
+
     def prune_heads(self, heads_to_prune):
         """ Prunes heads of the base model.
 
@@ -178,14 +208,12 @@ class PreTrainedModel(nn.Module):
                 heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`).
                 E.g. {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2.
         """
-        base_model = getattr(self, self.base_model_prefix, self)  # get the base model if needed
-
         # save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads
         for layer, heads in heads_to_prune.items():
             union_heads = set(self.config.pruned_heads.get(layer, [])) | set(heads)
             self.config.pruned_heads[layer] = list(union_heads)  # Unfortunately we have to store it as list for JSON
 
-        base_model._prune_heads(heads_to_prune)
+        self.base_model._prune_heads(heads_to_prune)
 
     def save_pretrained(self, save_directory):
         """ Save a model and its configuration file to a directory, so that it
diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py
index 166b98de63..396632d55e 100644
--- a/transformers/modeling_xlm.py
+++ b/transformers/modeling_xlm.py
@@ -407,10 +407,14 @@ class XLMModel(XLMPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        self.embeddings = self._get_resized_embeddings(self.embeddings, new_num_tokens)
+    @property
+    def input_embeddings(self):
         return self.embeddings
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.embeddings = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
@@ -618,12 +622,10 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
         self.pred_layer = XLMPredLayer(config)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the embeddings
-        """
-        self._tie_or_clone_weights(self.pred_layer.proj, self.transformer.embeddings)
+    @property
+    def output_embeddings(self):
+        return self.pred_layer.proj
 
     def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
                 lengths=None, cache=None, head_mask=None, labels=None):
diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py
index e191ebadd0..3173616eb8 100644
--- a/transformers/modeling_xlnet.py
+++ b/transformers/modeling_xlnet.py
@@ -611,10 +611,14 @@ class XLNetModel(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    def _resize_token_embeddings(self, new_num_tokens):
-        self.word_embedding = self._get_resized_embeddings(self.word_embedding, new_num_tokens)
+    @property
+    def input_embeddings(self):
         return self.word_embedding
 
+    @input_embeddings.setter
+    def input_embeddings(self, new_embeddings):
+        self.word_embedding = new_embeddings
+
     def _prune_heads(self, heads_to_prune):
         raise NotImplementedError
 
@@ -918,12 +922,10 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
         self.lm_loss = nn.Linear(config.d_model, config.n_token, bias=True)
 
         self.init_weights()
-        self.tie_weights()
 
-    def tie_weights(self):
-        """ Make sure we are sharing the embeddings
-        """
-        self._tie_or_clone_weights(self.lm_loss, self.transformer.word_embedding)
+    @property
+    def output_embeddings(self):
+        return self.lm_loss
 
     def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
                 token_type_ids=None, input_mask=None, head_mask=None, labels=None):
diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 008d7c0d51..300b019dfb 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -463,6 +463,15 @@ class CommonTestCases:
 
                 self.assertTrue(models_equal)
 
+        def test_model_common_attributes(self):
+            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+            for model_class in self.all_model_classes:
+                model = model_class(config)
+                self.assertTrue(hasattr(model, 'input_embeddings'))
+                setattr(model, 'input_embeddings', torch.nn.Embedding(10, 10))
+                self.assertTrue(hasattr(model, 'output_embeddings'))
+
         def test_tie_model_weights(self):
             if not self.test_torchscript:
                 return

From 1724cee8c4b62138b5f6968bda4668b3ad59968a Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 4 Nov 2019 15:34:10 +0100
Subject: [PATCH 122/269] switch from properties to methods

---
 templates/adding_a_new_model/modeling_xxx.py |  8 ++---
 transformers/modeling_bert.py                | 14 +++-----
 transformers/modeling_ctrl.py                |  9 ++---
 transformers/modeling_distilbert.py          |  9 ++---
 transformers/modeling_gpt2.py                | 12 +++----
 transformers/modeling_openai.py              | 12 +++----
 transformers/modeling_roberta.py             |  8 ++---
 transformers/modeling_transfo_xl.py          |  6 ++--
 transformers/modeling_utils.py               | 37 ++++++++++++++------
 transformers/modeling_xlm.py                 |  9 ++---
 transformers/modeling_xlnet.py               |  9 ++---
 transformers/tests/modeling_common_test.py   | 12 +++++--
 12 files changed, 70 insertions(+), 75 deletions(-)

diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
index 5335439dfb..ff64f13f40 100644
--- a/templates/adding_a_new_model/modeling_xxx.py
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -281,11 +281,10 @@ class XxxModel(XxxPreTrainedModel):
         self.init_weights()
 
     @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.embeddings.word_embeddings
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.embeddings.word_embeddings = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -382,8 +381,7 @@ class XxxForMaskedLM(XxxPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index a920aa86d3..6a702c251c 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -601,13 +601,11 @@ class BertModel(BertPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.embeddings.word_embeddings
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
-        self.embeddings.word_embeddings = new_embeddings
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
 
     def _prune_heads(self, heads_to_prune):
         """ Prunes heads of the model.
@@ -753,8 +751,7 @@ class BertForPreTraining(BertPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.cls.predictions.decoder
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
@@ -829,8 +826,7 @@ class BertForMaskedLM(BertPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.cls.predictions.decoder
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py
index c588dc30ba..1873040a8e 100644
--- a/transformers/modeling_ctrl.py
+++ b/transformers/modeling_ctrl.py
@@ -289,12 +289,10 @@ class CTRLModel(CTRLPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.w
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.w = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -454,8 +452,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head
 
     def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py
index 7365d1a7dc..989b4affae 100644
--- a/transformers/modeling_distilbert.py
+++ b/transformers/modeling_distilbert.py
@@ -421,12 +421,10 @@ class DistilBertModel(DistilBertPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.embeddings.word_embeddings
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.embeddings.word_embeddings = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -513,8 +511,7 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel):
 
         self.mlm_loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.vocab_projector
 
     def forward(self, input_ids, attention_mask=None, head_mask=None, masked_lm_labels=None):
diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index 9f48152884..4abe21d22b 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -357,12 +357,10 @@ class GPT2Model(GPT2PreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.wte
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.wte = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -519,8 +517,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head
 
     def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
@@ -623,8 +620,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head
 
     def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py
index 9e25b9cfb4..36c1560f1a 100644
--- a/transformers/modeling_openai.py
+++ b/transformers/modeling_openai.py
@@ -360,12 +360,10 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.tokens_embed
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.tokens_embed = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -494,8 +492,7 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
@@ -584,8 +581,7 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index 81216c93d4..af7c0b65d9 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -169,10 +169,11 @@ class RobertaModel(BertModel):
         self.embeddings = RobertaEmbeddings(config)
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.embeddings.word_embeddings
 
+    def set_input_embeddings(self, value):
+        self.embeddings.word_emebddings = value
 
 @add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """,
     ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
@@ -218,8 +219,7 @@ class RobertaForMaskedLM(BertPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_head.decoder
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py
index 0bc7fadd77..25b46cbc2c 100644
--- a/transformers/modeling_transfo_xl.py
+++ b/transformers/modeling_transfo_xl.py
@@ -639,12 +639,10 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.word_emb
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.word_emb = new_embeddings
 
     def backward_compatible(self):
diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index 3c2d8e199d..063f52365d 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -87,21 +87,37 @@ class PreTrainedModel(nn.Module):
     def base_model(self):
         return getattr(self, self.base_model_prefix, self)
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
+        """ Get model's input embeddings
+        """
         base_model = getattr(self, self.base_model_prefix, self)
-        return base_model.input_embeddings
+        if base_model is not self:
+            return base_model.get_input_embeddings()
+        else:
+            raise NotImplementedError
 
-    @property
-    def output_embeddings(self):
+    def set_input_embeddings(self, value):
+        """ Set model's input embeddings
+        """
+        base_model = getattr(self, self.base_model_prefix, self)
+        if base_model is not self:
+            base_model.set_input_embeddings(value)
+        else:
+            raise NotImplementedError
+
+    def get_output_embeddings(self):
+        """ Get model's output embeddings
+            Return None if the model doesn't have output embeddings
+        """
         return None  # Overwrite for models with output embeddings
 
     def tie_weights(self):
         """ Make sure we are sharing the input and output embeddings.
             Export to TorchScript can't handle parameter sharing so we are cloning them instead.
         """
-        if self.output_embeddings is not None:
-            self._tie_or_clone_weights(self.output_embeddings, self.input_embeddings)
+        output_embeddings = self.get_output_embeddings()
+        if output_embeddings is not None:
+            self._tie_or_clone_weights(output_embeddings, self.get_input_embeddings())
 
     def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
         """ Tie or clone module weights depending of weither we are using TorchScript or not
@@ -150,9 +166,10 @@ class PreTrainedModel(nn.Module):
         return model_embeds
 
     def _resize_token_embeddings(self, new_num_tokens):
-        old_embeddings = self.input_embeddings
-        self.input_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
-        return self.input_embeddings
+        old_embeddings = self.get_input_embeddings()
+        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
+        self.set_input_embeddings(new_embeddings)
+        return self.get_input_embeddings()
 
     def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None):
         """ Build a resized Embedding Module from a provided token Embedding Module.
diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py
index 396632d55e..80b7d1edbf 100644
--- a/transformers/modeling_xlm.py
+++ b/transformers/modeling_xlm.py
@@ -407,12 +407,10 @@ class XLMModel(XLMPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.embeddings
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.embeddings = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -623,8 +621,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.pred_layer.proj
 
     def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py
index 3173616eb8..0d94cdaf35 100644
--- a/transformers/modeling_xlnet.py
+++ b/transformers/modeling_xlnet.py
@@ -611,12 +611,10 @@ class XLNetModel(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def input_embeddings(self):
+    def get_input_embeddings(self):
         return self.word_embedding
 
-    @input_embeddings.setter
-    def input_embeddings(self, new_embeddings):
+    def set_input_embeddings(self, new_embeddings):
         self.word_embedding = new_embeddings
 
     def _prune_heads(self, heads_to_prune):
@@ -923,8 +921,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    @property
-    def output_embeddings(self):
+    def get_output_embeddings(self):
         return self.lm_loss
 
     def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 300b019dfb..5fb029f454 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -429,6 +429,12 @@ class CommonTestCases:
                     list(hidden_states[0].shape[-2:]),
                     [self.model_tester.seq_length, self.model_tester.hidden_size])
 
+        def test_debug(self):
+            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+            for model_class in self.all_model_classes:
+                model = model_class(config)
+                model_embed = model.resize_token_embeddings(config.vocab_size + 10)
+
         def test_resize_tokens_embeddings(self):
             original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
             if not self.test_resize_embeddings:
@@ -468,9 +474,9 @@ class CommonTestCases:
 
             for model_class in self.all_model_classes:
                 model = model_class(config)
-                self.assertTrue(hasattr(model, 'input_embeddings'))
-                setattr(model, 'input_embeddings', torch.nn.Embedding(10, 10))
-                self.assertTrue(hasattr(model, 'output_embeddings'))
+                model.get_input_embeddings()
+                model.set_input_embeddings(torch.nn.Embedding(10, 10))
+                model.get_output_embeddings()
 
         def test_tie_model_weights(self):
             if not self.test_torchscript:

From f02805da6fed33516fd77358068e459475c752ee Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 4 Nov 2019 15:42:23 +0100
Subject: [PATCH 123/269] fix tests

---
 transformers/tests/modeling_common_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 5fb029f454..159d9d85bb 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -492,11 +492,11 @@ class CommonTestCases:
                 return equal
 
             for model_class in self.all_model_classes:
-                if not hasattr(model_class, 'tie_weights'):
-                    continue
-
                 config.torchscript = True
                 model_not_tied = model_class(config)
+                if model_not_tied.get_output_embeddings() is None:
+                    continue
+
                 params_not_tied = list(model_not_tied.parameters())
 
                 config_tied = copy.deepcopy(config)

From b340a910ed50f0705975595c9b8c9ae26111c01d Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 4 Nov 2019 16:03:36 +0100
Subject: [PATCH 124/269] fix tests - flagged as slow all the tests downloading
 from AWS

---
 transformers/modeling_roberta.py                    | 2 +-
 transformers/tests/modeling_auto_test.py            | 4 ++++
 transformers/tests/modeling_common_test.py          | 8 ++------
 transformers/tests/modeling_encoder_decoder_test.py | 1 +
 transformers/tests/tokenization_auto_test.py        | 1 +
 transformers/tests/tokenization_bert_test.py        | 2 ++
 transformers/tests/tokenization_distilbert_test.py  | 2 ++
 transformers/tests/tokenization_roberta_test.py     | 2 ++
 transformers/tests/tokenization_utils_test.py       | 2 ++
 transformers/tests/tokenization_xlm_test.py         | 2 ++
 transformers/tests/tokenization_xlnet_test.py       | 2 ++
 11 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index af7c0b65d9..cbf285fa95 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -173,7 +173,7 @@ class RobertaModel(BertModel):
         return self.embeddings.word_embeddings
 
     def set_input_embeddings(self, value):
-        self.embeddings.word_emebddings = value
+        self.embeddings.word_embeddings = value
 
 @add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """,
     ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
diff --git a/transformers/tests/modeling_auto_test.py b/transformers/tests/modeling_auto_test.py
index af1de29cce..6d2c7ec979 100644
--- a/transformers/tests/modeling_auto_test.py
+++ b/transformers/tests/modeling_auto_test.py
@@ -38,6 +38,7 @@ else:
 
 
 class AutoModelTest(unittest.TestCase):
+    @pytest.mark.slow
     def test_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
@@ -52,6 +53,7 @@ class AutoModelTest(unittest.TestCase):
             for value in loading_info.values():
                 self.assertEqual(len(value), 0)
 
+    @pytest.mark.slow
     def test_lmhead_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
@@ -64,6 +66,7 @@ class AutoModelTest(unittest.TestCase):
             self.assertIsNotNone(model)
             self.assertIsInstance(model, BertForMaskedLM)
 
+    @pytest.mark.slow
     def test_sequence_classification_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
@@ -76,6 +79,7 @@ class AutoModelTest(unittest.TestCase):
             self.assertIsNotNone(model)
             self.assertIsInstance(model, BertForSequenceClassification)
 
+    @pytest.mark.slow
     def test_question_answering_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 159d9d85bb..ddc0f9f3de 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -429,12 +429,6 @@ class CommonTestCases:
                     list(hidden_states[0].shape[-2:]),
                     [self.model_tester.seq_length, self.model_tester.hidden_size])
 
-        def test_debug(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-            for model_class in self.all_model_classes:
-                model = model_class(config)
-                model_embed = model.resize_token_embeddings(config.vocab_size + 10)
-
         def test_resize_tokens_embeddings(self):
             original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
             if not self.test_resize_embeddings:
@@ -703,6 +697,7 @@ class CommonTestCases:
                 config_and_inputs = self.prepare_config_and_inputs()
                 self.create_and_check_presents(*config_and_inputs)
 
+        @pytest.mark.slow
         def run_slow_tests(self):
             self.create_and_check_model_from_pretrained()
 
@@ -776,6 +771,7 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None):
 
 
 class ModelUtilsTest(unittest.TestCase):
+    @pytest.mark.slow
     def test_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
diff --git a/transformers/tests/modeling_encoder_decoder_test.py b/transformers/tests/modeling_encoder_decoder_test.py
index 1ffd0ebc4c..a6c88ed9a9 100644
--- a/transformers/tests/modeling_encoder_decoder_test.py
+++ b/transformers/tests/modeling_encoder_decoder_test.py
@@ -27,6 +27,7 @@ else:
 
 
 class EncoderDecoderModelTest(unittest.TestCase):
+    @pytest.mark.slow
     def test_model2model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
diff --git a/transformers/tests/tokenization_auto_test.py b/transformers/tests/tokenization_auto_test.py
index 0f49ec75fb..79370811e8 100644
--- a/transformers/tests/tokenization_auto_test.py
+++ b/transformers/tests/tokenization_auto_test.py
@@ -26,6 +26,7 @@ from transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CON
 
 
 class AutoTokenizerTest(unittest.TestCase):
+    @pytest.mark.slow
     def test_tokenizer_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]:
diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py
index fd61ec30ba..73ea38e20a 100644
--- a/transformers/tests/tokenization_bert_test.py
+++ b/transformers/tests/tokenization_bert_test.py
@@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import os
 import unittest
+import pytest
 from io import open
 
 from transformers.tokenization_bert import (BasicTokenizer,
@@ -125,6 +126,7 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester):
         self.assertFalse(_is_punctuation(u"A"))
         self.assertFalse(_is_punctuation(u" "))
 
+    @pytest.mark.slow
     def test_sequence_builders(self):
         tokenizer = self.tokenizer_class.from_pretrained("bert-base-uncased")
 
diff --git a/transformers/tests/tokenization_distilbert_test.py b/transformers/tests/tokenization_distilbert_test.py
index e3c8376ca8..77a487651d 100644
--- a/transformers/tests/tokenization_distilbert_test.py
+++ b/transformers/tests/tokenization_distilbert_test.py
@@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import os
 import unittest
+import pytest
 from io import open
 
 from transformers.tokenization_distilbert import (DistilBertTokenizer)
@@ -30,6 +31,7 @@ class DistilBertTokenizationTest(BertTokenizationTest):
     def get_tokenizer(self, **kwargs):
         return DistilBertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
 
+    @pytest.mark.slow
     def test_sequence_builders(self):
         tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
 
diff --git a/transformers/tests/tokenization_roberta_test.py b/transformers/tests/tokenization_roberta_test.py
index b31dd94f21..a27bf7d654 100644
--- a/transformers/tests/tokenization_roberta_test.py
+++ b/transformers/tests/tokenization_roberta_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import os
 import json
 import unittest
+import pytest
 from io import open
 
 from transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES
@@ -78,6 +79,7 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester):
             [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]
         )
 
+    @pytest.mark.slow
     def test_sequence_builders(self):
         tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
 
diff --git a/transformers/tests/tokenization_utils_test.py b/transformers/tests/tokenization_utils_test.py
index cf55982c8f..8630191c69 100644
--- a/transformers/tests/tokenization_utils_test.py
+++ b/transformers/tests/tokenization_utils_test.py
@@ -18,11 +18,13 @@ from __future__ import print_function
 
 import unittest
 import six
+import pytest
 
 from transformers import PreTrainedTokenizer
 from transformers.tokenization_gpt2 import GPT2Tokenizer
 
 class TokenizerUtilsTest(unittest.TestCase):
+    @pytest.mark.slow
     def check_tokenizer_from_pretrained(self, tokenizer_class):
         s3_models = list(tokenizer_class.max_model_input_sizes.keys())
         for model_name in s3_models[:1]:
diff --git a/transformers/tests/tokenization_xlm_test.py b/transformers/tests/tokenization_xlm_test.py
index 567edf1ccd..3ff6564e34 100644
--- a/transformers/tests/tokenization_xlm_test.py
+++ b/transformers/tests/tokenization_xlm_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import os
 import unittest
 import json
+import pytest
 
 from transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES
 
@@ -66,6 +67,7 @@ class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester):
         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens)
 
+    @pytest.mark.slow
     def test_sequence_builders(self):
         tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048")
 
diff --git a/transformers/tests/tokenization_xlnet_test.py b/transformers/tests/tokenization_xlnet_test.py
index 653968b9af..2e14ffeb82 100644
--- a/transformers/tests/tokenization_xlnet_test.py
+++ b/transformers/tests/tokenization_xlnet_test.py
@@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import os
 import unittest
+import pytest
 
 from transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE)
 
@@ -89,6 +90,7 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester):
                                       u'9', u'2', u'0', u'0', u'0', u',', SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this',
                                       SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u'se', u'.'])
 
+    @pytest.mark.slow
     def test_sequence_builders(self):
         tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
 

From 89d627289890c8e556df72b7901260b038236e55 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Mon, 4 Nov 2019 16:21:12 +0100
Subject: [PATCH 125/269] Fix #1623

---
 .../distillation/run_squad_w_distillation.py  | 27 +++++++++++++------
 examples/run_bertology.py                     | 14 +++++++---
 examples/run_glue.py                          | 16 ++++++++---
 examples/run_lm_finetuning.py                 | 12 ++++++---
 examples/run_multiple_choice.py               | 14 +++++++---
 examples/run_ner.py                           | 12 ++++++---
 examples/run_squad.py                         | 12 ++++++---
 .../adding_a_new_example_script/run_xxx.py    | 12 ++++++---
 8 files changed, 87 insertions(+), 32 deletions(-)

diff --git a/examples/distillation/run_squad_w_distillation.py b/examples/distillation/run_squad_w_distillation.py
index a5194d0804..7c662df010 100644
--- a/examples/distillation/run_squad_w_distillation.py
+++ b/examples/distillation/run_squad_w_distillation.py
@@ -506,9 +506,15 @@ def main():
 
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.teacher_type is not None:
         assert args.teacher_name_or_path is not None
@@ -516,8 +522,11 @@ def main():
         assert args.alpha_ce + args.alpha_squad > 0.
         assert args.teacher_type != 'distilbert', "We constraint teachers not to be of type DistilBERT."
         teacher_config_class, teacher_model_class, _ = MODEL_CLASSES[args.teacher_type]
-        teacher_config = teacher_config_class.from_pretrained(args.teacher_name_or_path)
-        teacher = teacher_model_class.from_pretrained(args.teacher_name_or_path, config=teacher_config)
+        teacher_config = teacher_config_class.from_pretrained(args.teacher_name_or_path,
+                                                              cache_dir=args.cache_dir if args.cache_dir else None)
+        teacher = teacher_model_class.from_pretrained(args.teacher_name_or_path,
+                                                      config=teacher_config,
+                                                      cache_dir=args.cache_dir if args.cache_dir else None)
         teacher.to(args.device)
     else:
         teacher = None
@@ -553,8 +562,10 @@ def main():
         torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
 
         # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = model_class.from_pretrained(args.output_dir, cache_dir=args.cache_dir if args.cache_dir else None)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir,
+                                                    do_lower_case=args.do_lower_case,
+                                                    cache_dir=args.cache_dir if args.cache_dir else None)
         model.to(args.device)
 
 
@@ -571,7 +582,7 @@ def main():
         for checkpoint in checkpoints:
             # Reload the model
             global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
-            model = model_class.from_pretrained(checkpoint)
+            model = model_class.from_pretrained(checkpoint, cache_dir=args.cache_dir if args.cache_dir else None)
             model.to(args.device)
 
             # Evaluate
diff --git a/examples/run_bertology.py b/examples/run_bertology.py
index f37358359d..aae2d150d3 100644
--- a/examples/run_bertology.py
+++ b/examples/run_bertology.py
@@ -304,10 +304,16 @@ def main():
             break
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
     config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
-                                          num_labels=num_labels, finetuning_task=args.task_name,
-                                          output_attentions=True)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+                                          num_labels=num_labels,
+                                          finetuning_task=args.task_name,
+                                          output_attentions=True,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
diff --git a/examples/run_glue.py b/examples/run_glue.py
index 54f6689e4d..1558a812c3 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -477,9 +477,17 @@ def main():
 
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          num_labels=num_labels,
+                                          finetuning_task=args.task_name,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
@@ -514,7 +522,7 @@ def main():
 
         # Load a trained model and vocabulary that you have fine-tuned
         model = model_class.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
         model.to(args.device)
 
 
diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 982d8aa1b7..2044cfe9e8 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -471,12 +471,18 @@ def main():
         torch.distributed.barrier()  # Barrier to make sure only the first process in distributed training download model & vocab
 
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
     if args.block_size <= 0:
         args.block_size = tokenizer.max_len_single_sentence  # Our input block size will be the max possible for the model
     args.block_size = min(args.block_size, tokenizer.max_len_single_sentence)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
     model.to(args.device)
 
     if args.local_rank == 0:
diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py
index e11264cf57..638bbe74f1 100644
--- a/examples/run_multiple_choice.py
+++ b/examples/run_multiple_choice.py
@@ -464,9 +464,17 @@ def main():
 
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          num_labels=num_labels,
+                                          finetuning_task=args.task_name,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
diff --git a/examples/run_ner.py b/examples/run_ner.py
index aeb6a9a292..e3887ae453 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -428,11 +428,15 @@ def main():
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
     config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
-                                          num_labels=num_labels)
+                                          num_labels=num_labels,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
     tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
-                                                do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path),
-                                        config=config)
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool(".ckpt" in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
diff --git a/examples/run_squad.py b/examples/run_squad.py
index a263566c57..254dfbffc9 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -477,9 +477,15 @@ def main():
 
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
diff --git a/templates/adding_a_new_example_script/run_xxx.py b/templates/adding_a_new_example_script/run_xxx.py
index e348d9b5ea..489dcb19c7 100644
--- a/templates/adding_a_new_example_script/run_xxx.py
+++ b/templates/adding_a_new_example_script/run_xxx.py
@@ -472,9 +472,15 @@ def main():
 
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

From 68f7064a3ea979cdbdadfed62ad655eac4c53463 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Mon, 4 Nov 2019 11:52:35 -0500
Subject: [PATCH 126/269] Add `model.train()` line to ReadMe training example

Co-Authored-By: Santosh-Gupta <San.Gupta.ML@gmail.com>
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 6721163d16..40b08583b1 100644
--- a/README.md
+++ b/README.md
@@ -538,6 +538,7 @@ optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce
 scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler
 ### and used like this:
 for batch in train_data:
+    model.train()
     loss = model(batch)
     loss.backward()
     torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)

From 8e11de0e86124d61ae883f8ce06d71dfaed9b01f Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Fri, 1 Nov 2019 16:28:32 +0000
Subject: [PATCH 127/269] model forwards can take an inputs_embeds param

---
 transformers/modeling_gpt2.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index 4abe21d22b..6cbed33733 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -370,9 +370,15 @@ class GPT2Model(GPT2PreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.h[layer].attn.prune_heads(heads)
 
-    def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
-        input_shape = input_ids.size()
-        input_ids = input_ids.view(-1, input_shape[-1])
+    def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None):
+        if input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if token_type_ids is not None:
             token_type_ids = token_type_ids.view(-1, input_shape[-1])
         if position_ids is not None:
@@ -384,8 +390,9 @@ class GPT2Model(GPT2PreTrainedModel):
         else:
             past_length = past[0][0].size(-2)
         if position_ids is None:
-            position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device)
-            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
 
         # Attention mask.
         if attention_mask is not None:
@@ -419,7 +426,8 @@ class GPT2Model(GPT2PreTrainedModel):
         else:
             head_mask = [None] * self.config.n_layer
 
-        inputs_embeds = self.wte(input_ids)
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
         position_embeds = self.wpe(position_ids)
         if token_type_ids is not None:
             token_type_embeds = self.wte(token_type_ids)
@@ -520,14 +528,15 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                past=past,
                                                attention_mask=attention_mask,
                                                token_type_ids=token_type_ids,
                                                position_ids=position_ids,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
         hidden_states = transformer_outputs[0]
 
         lm_logits = self.lm_head(hidden_states)
@@ -623,14 +632,15 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 mc_token_ids=None, lm_labels=None, mc_labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                past=past,
                                                attention_mask=attention_mask,
                                                token_type_ids=token_type_ids,
                                                position_ids=position_ids,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         hidden_states = transformer_outputs[0]
 

From 9eddf44b7a3bc9373d3f1ed08f7ba2c16e8bf39c Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 4 Nov 2019 17:19:15 +0000
Subject: [PATCH 128/269] docstring + check

---
 transformers/modeling_gpt2.py     | 7 ++++++-
 transformers/modeling_tf_utils.py | 2 +-
 transformers/modeling_utils.py    | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index 6cbed33733..8a11cda084 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -313,6 +313,9 @@ GPT2_INPUTS_DOCSTRING = r"""    Inputs:
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want to input a probability distribution of tokens rather than actual tokens.
 """
 
 @add_start_docstrings("The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.",
@@ -371,7 +374,9 @@ class GPT2Model(GPT2PreTrainedModel):
             self.h[layer].attn.prune_heads(heads)
 
     def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None):
-        if input_ids is not None:
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
             input_shape = input_ids.size()
             input_ids = input_ids.view(-1, input_shape[-1])
         elif inputs_embeds is not None:
diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py
index a96e2765fd..f626327283 100644
--- a/transformers/modeling_tf_utils.py
+++ b/transformers/modeling_tf_utils.py
@@ -35,7 +35,7 @@ class TFPreTrainedModel(tf.keras.Model):
     r""" Base class for all TF models.
 
         :class:`~transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models
-        as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads.
+        as well as a few methods common to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads.
 
         Class attributes (overridden by derived classes):
             - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture.
diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index 063f52365d..91067a699b 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -53,7 +53,7 @@ class PreTrainedModel(nn.Module):
     r""" Base class for all models.
 
         :class:`~transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models
-        as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads.
+        as well as a few methods common to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads.
 
         Class attributes (overridden by derived classes):
             - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture.

From 00337e9687abd7bd3ff55609dc87f92fc4443aac Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 5 Nov 2019 00:39:18 +0000
Subject: [PATCH 129/269] [inputs_embeds] All PyTorch models

---
 .../adding_a_new_model/modeling_tf_xxx.py     |  4 +
 templates/adding_a_new_model/modeling_xxx.py  | 25 +++--
 transformers/modeling_bert.py                 | 95 ++++++++++++-------
 transformers/modeling_ctrl.py                 | 33 +++++--
 transformers/modeling_distilbert.py           | 37 ++++++--
 transformers/modeling_gpt2.py                 |  3 +-
 transformers/modeling_openai.py               | 43 +++++----
 transformers/modeling_roberta.py              | 47 ++++++---
 transformers/modeling_tf_bert.py              |  4 +
 transformers/modeling_tf_ctrl.py              |  4 +
 transformers/modeling_tf_distilbert.py        |  4 +
 transformers/modeling_tf_gpt2.py              |  4 +
 transformers/modeling_tf_openai.py            |  4 +
 transformers/modeling_tf_roberta.py           |  4 +
 transformers/modeling_tf_transfo_xl.py        |  4 +
 transformers/modeling_tf_xlm.py               |  4 +
 transformers/modeling_tf_xlnet.py             |  4 +
 transformers/modeling_transfo_xl.py           | 46 ++++++---
 transformers/modeling_xlm.py                  | 61 ++++++++----
 transformers/modeling_xlnet.py                | 65 +++++++++----
 transformers/tests/modeling_common_test.py    | 13 +++
 21 files changed, 361 insertions(+), 147 deletions(-)

diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py
index c661975768..faaece6f19 100644
--- a/templates/adding_a_new_model/modeling_tf_xxx.py
+++ b/templates/adding_a_new_model/modeling_tf_xxx.py
@@ -255,6 +255,10 @@ XXX_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare Xxx Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
index ff64f13f40..138ce70b2c 100644
--- a/templates/adding_a_new_model/modeling_xxx.py
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -238,6 +238,10 @@ XXX_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare Xxx Model transformer outputting raw hidden-states without any specific head on top.",
@@ -295,7 +299,7 @@ class XxxModel(XxxPreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.encoder.layer[layer].attention.prune_heads(heads)
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None):
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
         if token_type_ids is None:
@@ -449,14 +453,15 @@ class XxxForSequenceClassification(XxxPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
 
         outputs = self.transformer(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids, 
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         pooled_output = outputs[1]
 
@@ -520,14 +525,15 @@ class XxxForTokenClassification(XxxPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
 
         outputs = self.transformer(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids, 
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
 
@@ -603,14 +609,15 @@ class XxxForQuestionAnswering(XxxPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 start_positions=None, end_positions=None):
 
         outputs = self.transformer(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids, 
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
 
diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 6a702c251c..148bc2bd18 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -158,19 +158,26 @@ class BertEmbeddings(nn.Module):
         self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
 
-    def forward(self, input_ids, token_type_ids=None, position_ids=None):
-        seq_length = input_ids.size(1)
-        if position_ids is None:
-            position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
-            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
-        if token_type_ids is None:
-            token_type_ids = torch.zeros_like(input_ids)
+    def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None):
+        if input_ids is not None:
+            input_shape = input_ids.size()
+        else:
+            input_shape = inputs_embeds.size()[:-1]
 
-        words_embeddings = self.word_embeddings(input_ids)
+        seq_length = input_shape[1]
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+        if position_ids is None:
+            position_ids = torch.arange(seq_length, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0).expand(input_shape)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
         position_embeddings = self.position_embeddings(position_ids)
         token_type_embeddings = self.token_type_embeddings(token_type_ids)
 
-        embeddings = words_embeddings + position_embeddings + token_type_embeddings
+        embeddings = inputs_embeds + position_embeddings + token_type_embeddings
         embeddings = self.LayerNorm(embeddings)
         embeddings = self.dropout(embeddings)
         return embeddings
@@ -550,6 +557,10 @@ BERT_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
         **encoder_hidden_states**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``:
             Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model
             is configured as a decoder.
@@ -615,8 +626,8 @@ class BertModel(BertPreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.encoder.layer[layer].attention.prune_heads(heads)
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None,
-                head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None,
+                head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None):
         """ Forward pass on the Model.
 
         The model can behave as an encoder (with only self-attention) as well
@@ -632,12 +643,23 @@ class BertModel(BertPreTrainedModel):
             https://arxiv.org/abs/1706.03762
 
         """
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
         if attention_mask is None:
-            attention_mask = torch.ones_like(input_ids)
+            attention_mask = torch.ones(input_shape)
         if encoder_attention_mask is None:
-            encoder_attention_mask = torch.ones_like(input_ids)
+            encoder_attention_mask = torch.ones(input_shape)
         if token_type_ids is None:
-            token_type_ids = torch.zeros_like(input_ids)
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long)
 
         # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
         # ourselves in which case we just need to make it broadcastable to all heads.
@@ -649,8 +671,8 @@ class BertModel(BertPreTrainedModel):
         # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
         if attention_mask.dim() == 2:
             if self.config.is_decoder:
-                batch_size, seq_length = input_ids.size()
-                seq_ids = torch.arange(seq_length, device=input_ids.device)
+                batch_size, seq_length = input_shape
+                seq_ids = torch.arange(seq_length, device=device)
                 causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
                 extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
             else:
@@ -689,7 +711,7 @@ class BertModel(BertPreTrainedModel):
         else:
             head_mask = [None] * self.config.num_hidden_layers
 
-        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        embedding_output = self.embeddings(input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds)
         encoder_outputs = self.encoder(embedding_output,
                                        attention_mask=extended_attention_mask,
                                        head_mask=head_mask,
@@ -754,14 +776,15 @@ class BertForPreTraining(BertPreTrainedModel):
     def get_output_embeddings(self):
         return self.cls.predictions.decoder
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 masked_lm_labels=None, next_sentence_label=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         sequence_output, pooled_output = outputs[:2]
         prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)
@@ -829,7 +852,7 @@ class BertForMaskedLM(BertPreTrainedModel):
     def get_output_embeddings(self):
         return self.cls.predictions.decoder
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 masked_lm_labels=None, encoder_hidden_states=None, encoder_attention_mask=None, lm_labels=None, ):
 
         outputs = self.bert(input_ids,
@@ -837,6 +860,7 @@ class BertForMaskedLM(BertPreTrainedModel):
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
                             head_mask=head_mask,
+                            inputs_embeds=inputs_embeds,
                             encoder_hidden_states=encoder_hidden_states,
                             encoder_attention_mask=encoder_attention_mask)
 
@@ -908,14 +932,15 @@ class BertForNextSentencePrediction(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 next_sentence_label=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         pooled_output = outputs[1]
 
@@ -975,14 +1000,15 @@ class BertForSequenceClassification(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         pooled_output = outputs[1]
 
@@ -1049,8 +1075,8 @@ class BertForMultipleChoice(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
         num_choices = input_ids.shape[1]
 
         input_ids = input_ids.view(-1, input_ids.size(-1))
@@ -1062,7 +1088,8 @@ class BertForMultipleChoice(BertPreTrainedModel):
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         pooled_output = outputs[1]
 
@@ -1123,14 +1150,15 @@ class BertForTokenClassification(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
 
@@ -1207,14 +1235,15 @@ class BertForQuestionAnswering(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 start_positions=None, end_positions=None):
 
         outputs = self.bert(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids,
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
 
diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py
index 1873040a8e..405e33602a 100644
--- a/transformers/modeling_ctrl.py
+++ b/transformers/modeling_ctrl.py
@@ -236,6 +236,10 @@ CTRL_INPUTS_DOCSTRING = r"""    Inputs:
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.",
@@ -302,17 +306,26 @@ class CTRLModel(CTRLPreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.h[layer].attn.prune_heads(heads)
 
-    def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
-        input_shape = input_ids.size()
-        input_ids = input_ids.view(-1, input_shape[-1])
+    def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None):
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if past is None:
             past_length = 0
             past = [None] * len(self.h)
         else:
             past_length = past[0][0].size(-2)
         if position_ids is None:
-            position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device)
-            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
 
         # Attention mask.
         if attention_mask is not None:
@@ -354,9 +367,10 @@ class CTRLModel(CTRLPreTrainedModel):
             token_type_embeds = 0
         position_ids = position_ids.view(-1, input_shape[-1])
 
-        inputs_embeds = self.w(input_ids)
+        if inputs_embeds is None:
+            inputs_embeds = self.w(input_ids)
         # inputs_embeds = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded
-        seq_len = input_ids.shape[-1]
+        seq_len = input_shape[-1]
         mask = torch.triu(torch.ones(seq_len, seq_len), 1).to(inputs_embeds.device)
 
         inputs_embeds *= np.sqrt(self.d_model_size)
@@ -455,14 +469,15 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                past=past,
                                                attention_mask=attention_mask,
                                                token_type_ids=token_type_ids,
                                                position_ids=position_ids,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         hidden_states = transformer_outputs[0]
 
diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py
index 989b4affae..aca1670852 100644
--- a/transformers/modeling_distilbert.py
+++ b/transformers/modeling_distilbert.py
@@ -387,6 +387,10 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare DistilBERT encoder/transformer outputting raw hidden-states without any specific head on top.",
@@ -436,9 +440,18 @@ class DistilBertModel(DistilBertPreTrainedModel):
             self.transformer.layer[layer].attention.prune_heads(heads)
 
     def forward(self,
-                input_ids, attention_mask=None, head_mask=None):
+                input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None):
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if attention_mask is None:
-            attention_mask = torch.ones_like(input_ids) # (bs, seq_length)
+            attention_mask = torch.ones(input_shape) # (bs, seq_length)
 
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
@@ -455,8 +468,9 @@ class DistilBertModel(DistilBertPreTrainedModel):
         else:
             head_mask = [None] * self.config.num_hidden_layers
 
-        embedding_output = self.embeddings(input_ids)   # (bs, seq_length, dim)
-        tfmr_output = self.transformer(x=embedding_output,
+        if inputs_embeds is None:
+            inputs_embeds = self.embeddings(input_ids)   # (bs, seq_length, dim)
+        tfmr_output = self.transformer(x=inputs_embeds,
                                        attn_mask=attention_mask,
                                        head_mask=head_mask)
         hidden_state = tfmr_output[0]
@@ -514,10 +528,11 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel):
     def get_output_embeddings(self):
         return self.vocab_projector
 
-    def forward(self, input_ids, attention_mask=None, head_mask=None, masked_lm_labels=None):
+    def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, masked_lm_labels=None):
         dlbrt_output = self.distilbert(input_ids=input_ids,
                                        attention_mask=attention_mask,
-                                       head_mask=head_mask)
+                                       head_mask=head_mask,
+                                       inputs_embeds=inputs_embeds)
         hidden_states = dlbrt_output[0]                              # (bs, seq_length, dim)
         prediction_logits = self.vocab_transform(hidden_states)      # (bs, seq_length, dim)
         prediction_logits = gelu(prediction_logits)                  # (bs, seq_length, dim)
@@ -578,10 +593,11 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids,  attention_mask=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, labels=None):
         distilbert_output = self.distilbert(input_ids=input_ids,
                                             attention_mask=attention_mask,
-                                            head_mask=head_mask)
+                                            head_mask=head_mask,
+                                            inputs_embeds=inputs_embeds)
         hidden_state = distilbert_output[0]                    # (bs, seq_len, dim)
         pooled_output = hidden_state[:, 0]                    # (bs, dim)
         pooled_output = self.pre_classifier(pooled_output)   # (bs, dim)
@@ -652,10 +668,11 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel):
 
         self.init_weights()
         
-    def forward(self, input_ids, attention_mask=None, head_mask=None, start_positions=None, end_positions=None):
+    def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None):
         distilbert_output = self.distilbert(input_ids=input_ids,
                                             attention_mask=attention_mask,
-                                            head_mask=head_mask)
+                                            head_mask=head_mask,
+                                            inputs_embeds=inputs_embeds)
         hidden_states = distilbert_output[0]                                 # (bs, max_query_len, dim)
 
         hidden_states = self.dropout(hidden_states)                       # (bs, max_query_len, dim)
diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index 8a11cda084..6812b235f5 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -315,7 +315,8 @@ GPT2_INPUTS_DOCSTRING = r"""    Inputs:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
         **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
             Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
-            This is useful if you want to input a probability distribution of tokens rather than actual tokens.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py
index 36c1560f1a..246c3ae47b 100644
--- a/transformers/modeling_openai.py
+++ b/transformers/modeling_openai.py
@@ -322,6 +322,10 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""    Inputs:
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare OpenAI GPT transformer model outputting raw hidden-states without any specific head on top.",
@@ -373,14 +377,22 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.h[layer].attn.prune_heads(heads)
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None):
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if position_ids is None:
-            # This was used when we had a single embedding matrice from position and token embeddings
-            # start = self.config.vocab_size + self.config.n_special
-            # end = start + input_ids.size(-1)
-            # position_ids = torch.arange(start, end, dtype=torch.long, device=input_ids.device)
-            position_ids = torch.arange(input_ids.size(-1), dtype=torch.long, device=input_ids.device)
-            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
+            # Code is different from when we had a single embedding matrice from position and token embeddings
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(input_shape[-1], dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
 
         # Attention mask.
         if attention_mask is not None:
@@ -413,11 +425,8 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel):
         else:
             head_mask = [None] * self.config.n_layer
 
-        input_shape = input_ids.size()
-        input_ids = input_ids.view(-1, input_ids.size(-1))
-        position_ids = position_ids.view(-1, position_ids.size(-1))
-
-        inputs_embeds = self.tokens_embed(input_ids)
+        if inputs_embeds is None:
+            inputs_embeds = self.tokens_embed(input_ids)
         position_embeds = self.positions_embed(position_ids)
         if token_type_ids is not None:
             token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1))
@@ -495,13 +504,14 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                token_type_ids=token_type_ids,
                                                position_ids=position_ids,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
         hidden_states = transformer_outputs[0]
         lm_logits = self.lm_head(hidden_states)
 
@@ -584,13 +594,14 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 mc_token_ids=None, lm_labels=None, mc_labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                token_type_ids=token_type_ids,
                                                position_ids=position_ids,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
         hidden_states = transformer_outputs[0]
 
         lm_logits = self.lm_head(hidden_states)
diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index cbf285fa95..58b86000bb 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -48,16 +48,24 @@ class RobertaEmbeddings(BertEmbeddings):
         self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size,
                                                 padding_idx=self.padding_idx)
 
-    def forward(self, input_ids, token_type_ids=None, position_ids=None):
-        seq_length = input_ids.size(1)
+    def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None):
+        if input_ids is not None:
+            input_shape = input_ids.size()
+        else:
+            input_shape = inputs_embeds.size()[:-1]
+
+        seq_length = input_shape[1]
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
         if position_ids is None:
             # Position numbers begin at padding_idx+1. Padding symbols are ignored.
             # cf. fairseq's `utils.make_positions`
-            position_ids = torch.arange(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=torch.long, device=input_ids.device)
-            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
+            position_ids = torch.arange(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0).expand(input_shape)
         return super(RobertaEmbeddings, self).forward(input_ids,
                                                       token_type_ids=token_type_ids,
-                                                      position_ids=position_ids)
+                                                      position_ids=position_ids,
+                                                      inputs_embeds=inputs_embeds)
 
 
 ROBERTA_START_DOCSTRING = r"""    The RoBERTa model was proposed in
@@ -126,6 +134,10 @@ ROBERTA_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.",
@@ -222,13 +234,14 @@ class RobertaForMaskedLM(BertPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head.decoder
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 masked_lm_labels=None):
         outputs = self.roberta(input_ids,
                                attention_mask=attention_mask,
                                token_type_ids=token_type_ids,
                                position_ids=position_ids,
-                               head_mask=head_mask)
+                               head_mask=head_mask,
+                               inputs_embeds=inputs_embeds)
         sequence_output = outputs[0]
         prediction_scores = self.lm_head(sequence_output)
 
@@ -309,13 +322,14 @@ class RobertaForSequenceClassification(BertPreTrainedModel):
         self.roberta = RobertaModel(config)
         self.classifier = RobertaClassificationHead(config)
     
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 labels=None):
         outputs = self.roberta(input_ids,
                                attention_mask=attention_mask,
                                token_type_ids=token_type_ids,
                                position_ids=position_ids,
-                               head_mask=head_mask)
+                               head_mask=head_mask,
+                               inputs_embeds=inputs_embeds)
         sequence_output = outputs[0]
         logits = self.classifier(sequence_output)
 
@@ -372,6 +386,10 @@ class RobertaForMultipleChoice(BertPreTrainedModel):
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
         **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
             Labels for computing the multiple choice classification loss.
             Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension
@@ -415,8 +433,8 @@ class RobertaForMultipleChoice(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None,
-                position_ids=None, head_mask=None):
+    def forward(self, input_ids=None, token_type_ids=None, attention_mask=None, labels=None,
+                position_ids=None, head_mask=None, inputs_embeds=None):
         num_choices = input_ids.shape[1]
 
         flat_input_ids = input_ids.view(-1, input_ids.size(-1))
@@ -487,14 +505,15 @@ class RobertaForTokenClassification(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
 
         outputs = self.roberta(input_ids,
                                attention_mask=attention_mask,
                                token_type_ids=token_type_ids,
                                position_ids=position_ids,
-                               head_mask=head_mask)
+                               head_mask=head_mask,
+                               inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
 
diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py
index afe9b2946b..a1275db974 100644
--- a/transformers/modeling_tf_bert.py
+++ b/transformers/modeling_tf_bert.py
@@ -616,6 +616,10 @@ BERT_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py
index c8d181548b..dea590c5c5 100644
--- a/transformers/modeling_tf_ctrl.py
+++ b/transformers/modeling_tf_ctrl.py
@@ -374,6 +374,10 @@ CTRL_INPUTS_DOCSTRING = r"""    Inputs:
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py
index 188394816e..65acb9e142 100644
--- a/transformers/modeling_tf_distilbert.py
+++ b/transformers/modeling_tf_distilbert.py
@@ -508,6 +508,10 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare DistilBERT encoder/transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py
index 4188b273ba..50d58a6749 100644
--- a/transformers/modeling_tf_gpt2.py
+++ b/transformers/modeling_tf_gpt2.py
@@ -408,6 +408,10 @@ GPT2_INPUTS_DOCSTRING = r"""    Inputs:
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare GPT2 Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py
index 747c5171fd..18afa85dce 100644
--- a/transformers/modeling_tf_openai.py
+++ b/transformers/modeling_tf_openai.py
@@ -389,6 +389,10 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""    Inputs:
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare OpenAI GPT transformer model outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py
index 6f34081d3b..32abea659e 100644
--- a/transformers/modeling_tf_roberta.py
+++ b/transformers/modeling_tf_roberta.py
@@ -157,6 +157,10 @@ ROBERTA_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare RoBERTa Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py
index a3e403ce06..ec37aedd74 100644
--- a/transformers/modeling_tf_transfo_xl.py
+++ b/transformers/modeling_tf_transfo_xl.py
@@ -626,6 +626,10 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py
index 9ac5d28e1f..496d7d72a8 100644
--- a/transformers/modeling_tf_xlm.py
+++ b/transformers/modeling_tf_xlm.py
@@ -530,6 +530,10 @@ XLM_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare XLM Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py
index 8a25be78c1..bb33e45790 100644
--- a/transformers/modeling_tf_xlnet.py
+++ b/transformers/modeling_tf_xlnet.py
@@ -762,6 +762,10 @@ XLNET_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare XLNet Model transformer outputing raw hidden-states without any specific head on top.",
diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py
index 25b46cbc2c..a6a82f0dfe 100644
--- a/transformers/modeling_transfo_xl.py
+++ b/transformers/modeling_transfo_xl.py
@@ -553,6 +553,10 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.",
@@ -657,12 +661,12 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
         logger.info("Head pruning is not implemented for Transformer-XL model")
         pass
 
-    def init_mems(self, data):
+    def init_mems(self, bsz):
         if self.mem_len > 0:
             mems = []
             param = next(self.parameters())
             for i in range(self.n_layer):
-                empty = torch.zeros(self.mem_len, data.size(1), self.config.d_model,
+                empty = torch.zeros(self.mem_len, bsz, self.config.d_model,
                                     dtype=param.dtype, device=param.device)
                 mems.append(empty)
 
@@ -693,15 +697,22 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
 
         return new_mems
 
-    def forward(self, input_ids, mems=None, head_mask=None):
+    def forward(self, input_ids=None, mems=None, head_mask=None, inputs_embeds=None):
         # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library
         # so we transpose here from shape [bsz, len] to shape [len, bsz]
-        input_ids = input_ids.transpose(0, 1).contiguous()
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_ids = input_ids.transpose(0, 1).contiguous()
+            qlen, bsz = input_ids.size()
+        elif inputs_embeds is not None:
+            inputs_embeds = inputs_embeds.transpose(0, 1).contiguous()
+            qlen, bsz = inputs_embeds.shape[0], inputs_embeds.shape[1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
 
         if mems is None:
-            mems = self.init_mems(input_ids)
-
-        qlen, bsz = input_ids.size()
+            mems = self.init_mems(bsz)
 
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
@@ -718,7 +729,10 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
         else:
             head_mask = [None] * self.n_layer
 
-        word_emb = self.word_emb(input_ids)
+        if inputs_embeds is not None:
+            word_emb = inputs_embeds
+        else:
+            word_emb = self.word_emb(input_ids)
 
         mlen = mems[0].size(0) if mems is not None else 0
         klen = mlen + qlen
@@ -860,14 +874,18 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
     def reset_length(self, tgt_len, ext_len, mem_len):
         self.transformer.reset_length(tgt_len, ext_len, mem_len)
 
-    def init_mems(self, data):
-        return self.transformer.init_mems(data)
+    def init_mems(self, bsz):
+        return self.transformer.init_mems(bsz)
 
-    def forward(self, input_ids, mems=None, head_mask=None, labels=None):
-        bsz = input_ids.size(0)
-        tgt_len = input_ids.size(1)
+    def forward(self, input_ids=None, mems=None, head_mask=None, inputs_embeds=None, labels=None):
+        if input_ids is not None:
+            bsz, tgt_len = input_ids.size(0), input_ids.size(1)
+        elif inputs_embeds is not None:
+            bsz, tgt_len = inputs_embeds.size(0), inputs_embeds.size(1)
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
 
-        transformer_outputs = self.transformer(input_ids, mems=mems, head_mask=head_mask)
+        transformer_outputs = self.transformer(input_ids, mems=mems, head_mask=head_mask, inputs_embeds=inputs_embeds)
 
         last_hidden = transformer_outputs[0]
         pred_hid = last_hidden[:, -tgt_len:]
diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py
index 80b7d1edbf..257f0da394 100644
--- a/transformers/modeling_xlm.py
+++ b/transformers/modeling_xlm.py
@@ -311,6 +311,10 @@ XLM_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare XLM Model transformer outputting raw hidden-states without any specific head on top.",
@@ -421,14 +425,21 @@ class XLMModel(XLMPreTrainedModel):
         for layer, heads in heads_to_prune.items():
             self.attentions[layer].prune_heads(heads)
 
-    def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
-                lengths=None, cache=None, head_mask=None):  # removed: src_enc=None, src_len=None
+    def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
+                lengths=None, cache=None, head_mask=None, inputs_embeds=None):  # removed: src_enc=None, src_len=None
+        if input_ids is not None:
+            bs, slen = input_ids.size()
+        else:
+            bs, slen = inputs_embeds.size()[:-1]
+
         if lengths is None:
-            lengths = (input_ids != self.pad_index).sum(dim=1).long()
+            if input_ids is not None:
+                lengths = (input_ids != self.pad_index).sum(dim=1).long()
+            else:
+                lengths = torch.LongTensor([slen]*bs)
         # mask = input_ids != self.pad_index
 
         # check inputs
-        bs, slen = input_ids.size()
         assert lengths.size(0) == bs
         assert lengths.max().item() <= slen
         # input_ids = input_ids.transpose(0, 1)  # batch size as dimension 0
@@ -442,10 +453,12 @@ class XLMModel(XLMPreTrainedModel):
         # if self.is_decoder and src_enc is not None:
         #     src_mask = torch.arange(src_len.max(), dtype=torch.long, device=lengths.device) < src_len[:, None]
 
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
         # position_ids
         if position_ids is None:
-            position_ids = input_ids.new((slen,)).long()
-            position_ids = torch.arange(slen, out=position_ids).unsqueeze(0)
+            position_ids = torch.arange(slen, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0).expand((bs, slen))
         else:
             assert position_ids.size() == (bs, slen)  # (slen, bs)
             # position_ids = position_ids.transpose(0, 1)
@@ -471,7 +484,7 @@ class XLMModel(XLMPreTrainedModel):
             head_mask = [None] * self.n_layers
 
         # do not recompute cached elements
-        if cache is not None:
+        if cache is not None and input_ids is not None:
             _slen = slen - cache['slen']
             input_ids = input_ids[:, -_slen:]
             position_ids = position_ids[:, -_slen:]
@@ -481,8 +494,10 @@ class XLMModel(XLMPreTrainedModel):
             attn_mask = attn_mask[:, -_slen:]
 
         # embeddings
-        tensor = self.embeddings(input_ids)
-        tensor = tensor + self.position_embeddings(position_ids).expand_as(tensor)
+        if inputs_embeds is None:
+            inputs_embeds = self.embeddings(input_ids)
+
+        tensor = inputs_embeds + self.position_embeddings(position_ids).expand_as(inputs_embeds)
         if langs is not None and self.use_lang_emb:
             tensor = tensor + self.lang_embeddings(langs)
         if token_type_ids is not None:
@@ -624,8 +639,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
     def get_output_embeddings(self):
         return self.pred_layer.proj
 
-    def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
-                lengths=None, cache=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
+                lengths=None, cache=None, head_mask=None, inputs_embeds=None, labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                langs=langs,
@@ -633,7 +648,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
                                                position_ids=position_ids,
                                                lengths=lengths, 
                                                cache=cache,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         output = transformer_outputs[0]
         outputs = self.pred_layer(output, labels)
@@ -685,8 +701,8 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
-                lengths=None, cache=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
+                lengths=None, cache=None, head_mask=None, inputs_embeds=None, labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                langs=langs,
@@ -694,7 +710,8 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
                                                position_ids=position_ids,
                                                lengths=lengths, 
                                                cache=cache,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         output = transformer_outputs[0]
         logits = self.sequence_summary(output)
@@ -768,8 +785,8 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
-                lengths=None, cache=None, head_mask=None, start_positions=None, end_positions=None):
+    def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
+                lengths=None, cache=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                langs=langs,
@@ -777,7 +794,8 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
                                                position_ids=position_ids,
                                                lengths=lengths, 
                                                cache=cache,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         sequence_output = transformer_outputs[0]
 
@@ -863,8 +881,8 @@ class XLMForQuestionAnswering(XLMPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
-                lengths=None, cache=None, head_mask=None, start_positions=None, end_positions=None,
+    def forward(self, input_ids=None, attention_mask=None, langs=None, token_type_ids=None, position_ids=None,
+                lengths=None, cache=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None,
                 is_impossible=None, cls_index=None, p_mask=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
@@ -873,7 +891,8 @@ class XLMForQuestionAnswering(XLMPreTrainedModel):
                                                position_ids=position_ids,
                                                lengths=lengths, 
                                                cache=cache,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         output = transformer_outputs[0]
 
diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py
index 0d94cdaf35..658048a660 100644
--- a/transformers/modeling_xlnet.py
+++ b/transformers/modeling_xlnet.py
@@ -558,6 +558,10 @@ XLNET_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.",
@@ -712,19 +716,29 @@ class XLNetModel(XLNetPreTrainedModel):
         pos_emb = pos_emb.to(next(self.parameters()))
         return pos_emb
 
-    def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
-                token_type_ids=None, input_mask=None, head_mask=None):
+    def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
+                token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None):
         # the original code for XLNet uses shapes [len, bsz] with the batch dimension at the end
         # but we want a unified interface in the library with the batch size on the first dimension
         # so we move here the first dimension (batch) to the end
-        input_ids = input_ids.transpose(0, 1).contiguous()
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_ids = input_ids.transpose(0, 1).contiguous()
+            qlen, bsz = input_ids.shape[0], input_ids.shape[1]
+        elif inputs_embeds is not None:
+            inputs_embeds.transpose(0, 1).contiguous()
+            qlen, bsz = inputs_embeds.shape[0], inputs_embeds.shape[1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         token_type_ids = token_type_ids.transpose(0, 1).contiguous() if token_type_ids is not None else None
         input_mask = input_mask.transpose(0, 1).contiguous() if input_mask is not None else None
         attention_mask = attention_mask.transpose(0, 1).contiguous() if attention_mask is not None else None
         perm_mask = perm_mask.permute(1, 2, 0).contiguous() if perm_mask is not None else None
         target_mapping = target_mapping.permute(1, 2, 0).contiguous() if target_mapping is not None else None
 
-        qlen, bsz = input_ids.shape[0], input_ids.shape[1]
+
         mlen = mems[0].shape[0] if mems is not None and mems[0] is not None else 0
         klen = mlen + qlen
 
@@ -777,7 +791,10 @@ class XLNetModel(XLNetPreTrainedModel):
             non_tgt_mask = None
 
         ##### Word embeddings and prepare h & g hidden states
-        word_emb_k = self.word_embedding(input_ids)
+        if inputs_embeds is not None:
+            word_emb_k = inputs_embeds
+        else:
+            word_emb_k = self.word_embedding(input_ids)
         output_h = self.dropout(word_emb_k)
         if target_mapping is not None:
             word_emb_q = self.mask_emb.expand(target_mapping.shape[0], bsz, -1)
@@ -924,8 +941,8 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_loss
 
-    def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
-                token_type_ids=None, input_mask=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
+                token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                mems=mems,
@@ -933,7 +950,8 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
                                                target_mapping=target_mapping,
                                                token_type_ids=token_type_ids,
                                                input_mask=input_mask,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
 
         logits = self.lm_loss(transformer_outputs[0])
 
@@ -998,8 +1016,8 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
-                token_type_ids=None, input_mask=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
+                token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, labels=None):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
                                                mems=mems,
@@ -1007,7 +1025,8 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel):
                                                target_mapping=target_mapping,
                                                token_type_ids=token_type_ids,
                                                input_mask=input_mask,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
         output = transformer_outputs[0]
 
         output = self.sequence_summary(output)
@@ -1049,6 +1068,10 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel):
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
         **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
             Labels for computing the multiple choice classification loss.
             Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension
@@ -1093,9 +1116,9 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, token_type_ids=None, input_mask=None, attention_mask=None,
+    def forward(self, input_ids=None, token_type_ids=None, input_mask=None, attention_mask=None,
                 mems=None, perm_mask=None, target_mapping=None,
-                labels=None, head_mask=None):
+                labels=None, head_mask=None, inputs_embeds=None):
         num_choices = input_ids.shape[1]
 
         flat_input_ids = input_ids.view(-1, input_ids.size(-1))
@@ -1106,7 +1129,7 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel):
         transformer_outputs = self.transformer(flat_input_ids, token_type_ids=flat_token_type_ids,
                                                input_mask=flat_input_mask, attention_mask=flat_attention_mask,
                                                mems=mems, perm_mask=perm_mask, target_mapping=target_mapping,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask, inputs_embeds=inputs_embeds)
 
 
         output = transformer_outputs[0]
@@ -1178,8 +1201,8 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
-                token_type_ids=None, input_mask=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
+                token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None,
                 start_positions=None, end_positions=None):
 
         outputs = self.transformer(input_ids,
@@ -1189,7 +1212,8 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel):
                                     target_mapping=target_mapping,
                                     token_type_ids=token_type_ids,
                                     input_mask=input_mask,
-                                    head_mask=head_mask)
+                                    head_mask=head_mask,
+                                    inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
 
@@ -1294,8 +1318,8 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
-                token_type_ids=None, input_mask=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
+                token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None,
                 start_positions=None, end_positions=None, is_impossible=None, cls_index=None, p_mask=None,):
         transformer_outputs = self.transformer(input_ids,
                                                attention_mask=attention_mask,
@@ -1304,7 +1328,8 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel):
                                                target_mapping=target_mapping,
                                                token_type_ids=token_type_ids,
                                                input_mask=input_mask,
-                                               head_mask=head_mask)
+                                               head_mask=head_mask,
+                                               inputs_embeds=inputs_embeds)
         hidden_states = transformer_outputs[0]
         start_logits = self.start_logits(hidden_states, p_mask=p_mask)
 
diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index ddc0f9f3de..38b2ceafa4 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -525,6 +525,19 @@ class CommonTestCases:
                 # self.assertTrue(model.transformer.wte.weight.shape, model.lm_head.weight.shape)
                 # self.assertTrue(check_same_values(model.transformer.wte, model.lm_head))
 
+        def test_inputs_embeds(self):
+            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+            input_ids = inputs_dict["input_ids"]
+            del inputs_dict["input_ids"]
+
+            for model_class in self.all_model_classes:
+                model = model_class(config)
+                model.eval()
+
+                wte = model.get_input_embeddings()
+                inputs_dict["inputs_embeds"] = wte(input_ids)
+                outputs = model(**inputs_dict)
+
 
     class GPTModelTester(CommonModelTester):
 

From d7906165a329c17e6e49d5069e9b21fa37d50773 Mon Sep 17 00:00:00 2001
From: Oren Amsalem <oren.amsalem1@mail.huji.ac.il>
Date: Tue, 5 Nov 2019 10:34:27 +0200
Subject: [PATCH 130/269] add progress bar for convert_examples_to_features

It takes considerate amount of time (~10 min) to parse the examples to features, it is good to have a progress-bar to track this
---
 examples/utils_squad.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/utils_squad.py b/examples/utils_squad.py
index 6d1c86493d..c04dacf6d3 100644
--- a/examples/utils_squad.py
+++ b/examples/utils_squad.py
@@ -23,6 +23,7 @@ import logging
 import math
 import collections
 from io import open
+from tqdm import tqdm
 
 from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
 
@@ -202,7 +203,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
     # f = np.zeros((max_N, max_M), dtype=np.float32)
 
     features = []
-    for (example_index, example) in enumerate(examples):
+    for (example_index, example) in enumerate(tqdm(examples)):
 
         # if example_index % 100 == 0:
         #     logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)

From f1e4db2aa80d72bfe9992476ccca52348a789db0 Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Tue, 5 Nov 2019 09:38:00 +0100
Subject: [PATCH 131/269] Fix #1686

---
 transformers/modeling_openai.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py
index 36c1560f1a..12c0f82e60 100644
--- a/transformers/modeling_openai.py
+++ b/transformers/modeling_openai.py
@@ -565,9 +565,12 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
         tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt')
         model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt')
         tokenizer.add_special_tokens({'cls_token': '[CLS]'})  # Add a [CLS] to the vocabulary (we should train it also!)
+        model.resize_token_embeddings(len(tokenizer))
+
         choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"]
         input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0)  # Batch size 1, 2 choices
-        mc_token_ids = torch.tensor([input_ids.size(-1), input_ids.size(-1)]).unsqueeze(0)  # Batch size 1
+        mc_token_ids = torch.tensor([input_ids.size(-1)-1, input_ids.size(-1)-1]).unsqueeze(0)  # Batch size 1
+
         outputs = model(input_ids, mc_token_ids=mc_token_ids)
         lm_prediction_scores, mc_prediction_scores = outputs[:2]
 

From a44f112fb9cf8b2b5c0c710308820ff18f80a303 Mon Sep 17 00:00:00 2001
From: Clement <clementdelangue@gmail.Com>
Date: Tue, 5 Nov 2019 08:48:26 -0500
Subject: [PATCH 132/269] add authors for models

---
 .github/ISSUE_TEMPLATE/--new-model-addition.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/ISSUE_TEMPLATE/--new-model-addition.md b/.github/ISSUE_TEMPLATE/--new-model-addition.md
index 96fd85269d..456151d2a9 100644
--- a/.github/ISSUE_TEMPLATE/--new-model-addition.md
+++ b/.github/ISSUE_TEMPLATE/--new-model-addition.md
@@ -17,6 +17,7 @@ assignees: ''
 
 * [ ] the model implementation is available: (give details)
 * [ ] the model weights are available: (give details)
+* [ ] who are the authors: (mention them)
 
 ## Additional context
 

From d7d36181fdefdabadc53adf51bed4a2680f5880a Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 5 Nov 2019 17:49:01 +0000
Subject: [PATCH 133/269] GPT-2 XL

---
 docs/source/pretrained_models.rst  | 3 +++
 transformers/configuration_gpt2.py | 1 +
 transformers/modeling_gpt2.py      | 1 +
 transformers/tokenization_gpt2.py  | 3 +++
 4 files changed, 8 insertions(+)

diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index 43c08228bd..559c81cbb0 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -73,6 +73,9 @@ Here is the full list of the currently provided pretrained models together with
 |                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``gpt2-large``                                             | | 36-layer, 1280-hidden, 20-heads, 774M parameters.                                                                                   |
 |                   |                                                            | | OpenAI's Large-sized GPT-2 English model                                                                                            |
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``gpt2-xl``                                                | | 48-layer, 1600-hidden, 25-heads, 1558M parameters.                                                                                  |
+|                   |                                                            | | OpenAI's XL-sized GPT-2 English model                                                                                               |
 +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 | Transformer-XL    | ``transfo-xl-wt103``                                       | | 18-layer, 1024-hidden, 16-heads, 257M parameters.                                                                                   |
 |                   |                                                            | | English model trained on wikitext-103                                                                                               |
diff --git a/transformers/configuration_gpt2.py b/transformers/configuration_gpt2.py
index e7d853f317..c2fb4948d3 100644
--- a/transformers/configuration_gpt2.py
+++ b/transformers/configuration_gpt2.py
@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
 GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json",
                                       "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json",
                                       "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json",
+                                      "gpt2-xl": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-config.json",
                                       "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-config.json",}
 
 class GPT2Config(PretrainedConfig):
diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index 6812b235f5..e3d26797c8 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -39,6 +39,7 @@ logger = logging.getLogger(__name__)
 GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin",
                                      "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-pytorch_model.bin",
                                      "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-pytorch_model.bin",
+                                     "gpt2-xl": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-pytorch_model.bin",
                                      "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-pytorch_model.bin",}
 
 def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path):
diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py
index 6a7f75acb2..4bec515903 100644
--- a/transformers/tokenization_gpt2.py
+++ b/transformers/tokenization_gpt2.py
@@ -46,6 +46,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
         'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json",
         'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json",
         'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-vocab.json",
+        'gpt2-xl': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-vocab.json",
         'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-vocab.json",
     },
     'merges_file':
@@ -53,6 +54,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
         'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt",
         'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt",
         'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-merges.txt",
+        'gpt2-xl': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-xl-merges.txt",
         'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-merges.txt",
     },
 }
@@ -61,6 +63,7 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
     'gpt2': 1024,
     'gpt2-medium': 1024,
     'gpt2-large': 1024,
+    'gpt2-xl': 1024,
     'distilgpt2': 1024,
 }
 

From de890ae67d43e1e5d031a815dab5dfed081e9a95 Mon Sep 17 00:00:00 2001
From: Dom Hudson <dom.hudson@thoughtriver.com>
Date: Tue, 5 Nov 2019 11:04:59 +0000
Subject: [PATCH 134/269] Updating docblocks in optimizers.py

---
 transformers/optimization.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/transformers/optimization.py b/transformers/optimization.py
index 39dc7a50ff..a48b5fea54 100644
--- a/transformers/optimization.py
+++ b/transformers/optimization.py
@@ -32,8 +32,9 @@ class ConstantLRSchedule(LambdaLR):
 
 class WarmupConstantSchedule(LambdaLR):
     """ Linear warmup and then constant.
-        Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps.
-        Keeps learning rate schedule equal to 1. after warmup_steps.
+        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
+        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
+        Keeps multiplicative variable equal to 1. after warmup_steps.
     """
     def __init__(self, optimizer, warmup_steps, last_epoch=-1):
         self.warmup_steps = warmup_steps
@@ -47,8 +48,9 @@ class WarmupConstantSchedule(LambdaLR):
 
 class WarmupLinearSchedule(LambdaLR):
     """ Linear warmup and then linear decay.
-        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
-        Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps.
+        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
+        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
+        Linearly decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps.
     """
     def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1):
         self.warmup_steps = warmup_steps
@@ -63,9 +65,10 @@ class WarmupLinearSchedule(LambdaLR):
 
 class WarmupCosineSchedule(LambdaLR):
     """ Linear warmup and then cosine decay.
-        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
-        Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.
-        If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.
+        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
+        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
+        Decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.
+        If `cycles` (default=0.5) is different from default, then the multiplicative variable follows cosine function after warmup.
     """
     def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):
         self.warmup_steps = warmup_steps
@@ -83,8 +86,9 @@ class WarmupCosineSchedule(LambdaLR):
 
 class WarmupCosineWithHardRestartsSchedule(LambdaLR):
     """ Linear warmup and then cosine cycles with hard restarts.
-        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
-        If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying
+        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
+        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
+        If `cycles` (default=1.) is different from default, learning rate  follows `cycles` times a cosine decaying
         learning rate (with hard restarts).
     """
     def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1):

From 30968d70afedb1a9815164737cdc3779f2f058fe Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 5 Nov 2019 19:06:12 -0500
Subject: [PATCH 135/269] misc doc

---
 docs/source/serialization.rst                 |  2 +-
 examples/contrib/run_openai_gpt.py            |  2 +-
 templates/adding_a_new_model/README.md        | 14 ++++++------
 .../adding_a_new_model/modeling_tf_xxx.py     |  2 +-
 templates/adding_a_new_model/modeling_xxx.py  | 22 ++++++++++++++-----
 transformers/modeling_utils.py                |  2 +-
 6 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst
index c948104d69..d2862dc0b5 100644
--- a/docs/source/serialization.rst
+++ b/docs/source/serialization.rst
@@ -106,7 +106,7 @@ This section explain how you can save and re-load a fine-tuned model (BERT, GPT,
 There are three types of files you need to save to be able to reload a fine-tuned model:
 
 
-* the model it-self which should be saved following PyTorch serialization `best practices <https://pytorch.org/docs/stable/notes/serialization.html#best-practices>`__\ ,
+* the model itself which should be saved following PyTorch serialization `best practices <https://pytorch.org/docs/stable/notes/serialization.html#best-practices>`__\ ,
 * the configuration file of the model which is saved as a JSON file, and
 * the vocabulary (and the merges for the BPE-based models GPT and GPT-2).
 
diff --git a/examples/contrib/run_openai_gpt.py b/examples/contrib/run_openai_gpt.py
index 661c1c305b..7eb1b0be76 100644
--- a/examples/contrib/run_openai_gpt.py
+++ b/examples/contrib/run_openai_gpt.py
@@ -237,7 +237,7 @@ def main():
     # Save a trained model
     if args.do_train:
         # Save a trained model, configuration and tokenizer
-        model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self
+        model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model itself
 
         # If we save using the predefined names, we can load using `from_pretrained`
         output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
diff --git a/templates/adding_a_new_model/README.md b/templates/adding_a_new_model/README.md
index 1569b51e89..b546555bd6 100644
--- a/templates/adding_a_new_model/README.md
+++ b/templates/adding_a_new_model/README.md
@@ -7,7 +7,7 @@ The library is designed to incorporate a variety of models and code bases. As su
 One important point though is that the library has the following goals impacting the way models are incorporated:
 
 - one specific feature of the API is the capability to run the model and tokenizer inline. The tokenization code thus often have to be slightly adapted to allow for running in the python interpreter.
-- the package is also designed to be as self-consistent and with a small and reliable set of packages dependencies. In consequence, additional dependencies are usually not allowed when adding a model but can be allowed for the inclusion of a new tokenizer (recent examples of dependencies added for tokenizer specificites includes `sentencepiece` and `sacremoses`). Please make sure to check the existing dependencies when possible before adding a new one.
+- the package is also designed to be as self-consistent and with a small and reliable set of packages dependencies. In consequence, additional dependencies are usually not allowed when adding a model but can be allowed for the inclusion of a new tokenizer (recent examples of dependencies added for tokenizer specificities include `sentencepiece` and `sacremoses`). Please make sure to check the existing dependencies when possible before adding a new one.
 
 For a quick overview of the library organization, please check the [QuickStart section of the documentation](https://huggingface.co/transformers/quickstart.html).
 
@@ -20,7 +20,7 @@ Here an overview of the general workflow:
 - [ ] add tests
 - [ ] finalize
 
-Let's details what should be done at each step
+Let's detail what should be done at each step
 
 ## Adding model/configuration/tokenization classes
 
@@ -28,16 +28,16 @@ Here is the workflow for adding model/configuration/tokenization classes:
 
 - [ ] copy the python files from the present folder to the main folder and rename them, replacing `xxx` with your model name,
 - [ ] edit the files to replace `XXX` (with various casing) with your model name
-- [ ] copy-past or create a simple configuration class for your model in the `configuration_...` file
-- [ ] copy-past or create the code for your model in the `modeling_...` files (PyTorch and TF 2.0)
-- [ ] copy-past or create a tokenizer class for your model in the `tokenization_...` file
+- [ ] copy-paste or create a simple configuration class for your model in the `configuration_...` file
+- [ ] copy-paste or create the code for your model in the `modeling_...` files (PyTorch and TF 2.0)
+- [ ] copy-paste or create a tokenizer class for your model in the `tokenization_...` file
 
 # Adding conversion scripts
 
 Here is the workflow for the conversion scripts:
 
 - [ ] copy the conversion script (`convert_...`) from the present folder to the main folder.
-- [ ] edit this scipt to convert your original checkpoint weights to the current pytorch ones.
+- [ ] edit this script to convert your original checkpoint weights to the current pytorch ones.
 
 # Adding tests:
 
@@ -58,5 +58,5 @@ You can then finish the addition step by adding imports for your classes in the
 - [ ] add your models and tokenizer to `pipeline.py`
 - [ ] add a link to your conversion script in the main conversion utility (currently in `__main__` but will be moved to the `commands` subfolder in the near future)
 - [ ] edit the PyTorch to TF 2.0 conversion script to add your model in the `convert_pytorch_checkpoint_to_tf2.py` file
-- [ ] add a mention of your model in the doc: `README.md` and the documentation it-self at `docs/source/pretrained_models.rst`.
+- [ ] add a mention of your model in the doc: `README.md` and the documentation itself at `docs/source/pretrained_models.rst`.
 - [ ] upload the pretrained weigths, configurations and vocabulary files.
diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py
index faaece6f19..f1d898b47a 100644
--- a/templates/adding_a_new_model/modeling_tf_xxx.py
+++ b/templates/adding_a_new_model/modeling_tf_xxx.py
@@ -49,7 +49,7 @@ TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP = {
 ####################################################
 # TF 2.0 Models are constructed using Keras imperative API by sub-classing
 # - tf.keras.layers.Layer for the layers and
-# - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model)
+# - TFPreTrainedModel for the models (itself a sub-class of tf.keras.Model)
 ####################################################
 
 ####################################################
diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
index 138ce70b2c..d023f565f5 100644
--- a/templates/adding_a_new_model/modeling_xxx.py
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -120,7 +120,7 @@ def load_tf_weights_in_xxx(model, config, tf_checkpoint_path):
 ####################################################
 # PyTorch Models are constructed by sub-classing
 # - torch.nn.Module for the layers and
-# - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module)
+# - PreTrainedModel for the models (itself a sub-class of torch.nn.Module)
 ####################################################
 
 ####################################################
@@ -300,10 +300,19 @@ class XxxModel(XxxPreTrainedModel):
             self.encoder.layer[layer].attention.prune_heads(heads)
 
     def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None):
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if attention_mask is None:
-            attention_mask = torch.ones_like(input_ids)
+            attention_mask = torch.ones(input_shape)
         if token_type_ids is None:
-            token_type_ids = torch.zeros_like(input_ids)
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long)
 
         # We create a 3D attention mask from a 2D tensor mask.
         # Sizes are [batch_size, 1, 1, to_seq_length]
@@ -337,7 +346,7 @@ class XxxModel(XxxPreTrainedModel):
 
         ##################################
         # Replace this with your model code
-        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        embedding_output = self.embeddings(input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds)
         encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask)
         sequence_output = encoder_outputs[0]
         outputs = (sequence_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
@@ -388,14 +397,15 @@ class XxxForMaskedLM(XxxPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
                 masked_lm_labels=None):
 
         outputs = self.transformer(input_ids,
                             attention_mask=attention_mask,
                             token_type_ids=token_type_ids,
                             position_ids=position_ids, 
-                            head_mask=head_mask)
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
 
         sequence_output = outputs[0]
         prediction_scores = self.cls(sequence_output)
diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index 91067a699b..d51eefab58 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -238,7 +238,7 @@ class PreTrainedModel(nn.Module):
         """
         assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved"
 
-        # Only save the model it-self if we are using distributed training
+        # Only save the model itself if we are using distributed training
         model_to_save = self.module if hasattr(self, 'module') else self
 
         # Save configuration file

From f88c104d8f79e78a98c8ce6c1f4a78db73142eab Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 5 Nov 2019 19:56:43 -0500
Subject: [PATCH 136/269] [run_tf_glue] Add comment for context

---
 examples/run_tf_glue.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py
index 8878ce726e..54282277d2 100644
--- a/examples/run_tf_glue.py
+++ b/examples/run_tf_glue.py
@@ -73,6 +73,8 @@ model.save_pretrained('./save/')
 
 if TASK == "mrpc":
     # Load the TensorFlow model in PyTorch for inspection
+    # This is to demo the interoperability between the two frameworks, you don't have to 
+    # do this in real life (you can run the inference on the TF model).
     pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True)
 
     # Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task

From 13d9135fa5ba628b4b40891ee13b73d1880ae78d Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Wed, 6 Nov 2019 14:44:00 +0000
Subject: [PATCH 137/269] [tests] get rid of warning

cf. https://docs.pytest.org/en/latest/example/simple.html
---
 transformers/tests/conftest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/transformers/tests/conftest.py b/transformers/tests/conftest.py
index 841ebc8df9..44cfe3f464 100644
--- a/transformers/tests/conftest.py
+++ b/transformers/tests/conftest.py
@@ -9,6 +9,10 @@ def pytest_addoption(parser):
     )
 
 
+def pytest_configure(config):
+    config.addinivalue_line("markers", "slow: mark test as slow to run")
+
+
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--runslow"):
         # --runslow given in cli: do not skip slow tests

From 27e015bd546cc971af2413f36148421b22667ee7 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Wed, 6 Nov 2019 15:56:12 +0000
Subject: [PATCH 138/269] [tests] Flag to test on cuda

---
 transformers/tests/conftest.py           |  8 ++++++++
 transformers/tests/modeling_bert_test.py | 21 ++++++++++++++-------
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/transformers/tests/conftest.py b/transformers/tests/conftest.py
index 44cfe3f464..f809234cd5 100644
--- a/transformers/tests/conftest.py
+++ b/transformers/tests/conftest.py
@@ -7,6 +7,9 @@ def pytest_addoption(parser):
     parser.addoption(
         "--runslow", action="store_true", default=False, help="run slow tests"
     )
+    parser.addoption(
+        "--use_cuda", action="store_true", default=False, help="run tests on gpu"
+    )
 
 
 def pytest_configure(config):
@@ -21,3 +24,8 @@ def pytest_collection_modifyitems(config, items):
     for item in items:
         if "slow" in item.keywords:
             item.add_marker(skip_slow)
+
+@pytest.fixture
+def use_cuda(request):
+    """ Run test on gpu """
+    return request.config.getoption("--use_cuda")
diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py
index 67be910a7e..6c93c9a187 100644
--- a/transformers/tests/modeling_bert_test.py
+++ b/transformers/tests/modeling_bert_test.py
@@ -35,6 +35,7 @@ else:
     pytestmark = pytest.mark.skip("Require Torch")
 
 
+@pytest.mark.usefixtures("use_cuda")
 class BertModelTest(CommonTestCases.CommonModelTester):
 
     all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction,
@@ -66,6 +67,7 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                      num_labels=3,
                      num_choices=4,
                      scope=None,
+                     device='cpu',
                      ):
             self.parent = parent
             self.batch_size = batch_size
@@ -89,25 +91,26 @@ class BertModelTest(CommonTestCases.CommonModelTester):
             self.num_labels = num_labels
             self.num_choices = num_choices
             self.scope = scope
+            self.device = device
 
         def prepare_config_and_inputs(self):
-            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
+            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).to(self.device)
 
             input_mask = None
             if self.use_input_mask:
-                input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
+                input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2).to(self.device)
 
             token_type_ids = None
             if self.use_token_type_ids:
-                token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
+                token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size).to(self.device)
 
             sequence_labels = None
             token_labels = None
             choice_labels = None
             if self.use_labels:
-                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
-                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
-                choice_labels = ids_tensor([self.batch_size], self.num_choices)
+                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size).to(self.device)
+                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels).to(self.device)
+                choice_labels = ids_tensor([self.batch_size], self.num_choices).to(self.device)
 
             config = BertConfig(
                 vocab_size_or_config_json_file=self.vocab_size,
@@ -141,6 +144,7 @@ class BertModelTest(CommonTestCases.CommonModelTester):
 
         def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = BertModel(config=config)
+            model.to(input_ids.device)
             model.eval()
             sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
@@ -309,7 +313,10 @@ class BertModelTest(CommonTestCases.CommonModelTester):
     def test_config(self):
         self.config_tester.run_common_tests()
 
-    def test_bert_model(self):
+    def test_bert_model(self, use_cuda=False):
+        # ^^ This could be a real fixture
+        if use_cuda:
+            self.model_tester.device = "cuda"
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_bert_model(*config_and_inputs)
 

From d5319793c47326655cf25025ceb13a97afa00aad Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Wed, 6 Nov 2019 16:21:00 +0000
Subject: [PATCH 139/269] Fix BERT

---
 transformers/modeling_bert.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 148bc2bd18..7c2c6f4602 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -170,7 +170,7 @@ class BertEmbeddings(nn.Module):
             position_ids = torch.arange(seq_length, dtype=torch.long, device=device)
             position_ids = position_ids.unsqueeze(0).expand(input_shape)
         if token_type_ids is None:
-            token_type_ids = torch.zeros(input_shape, dtype=torch.long)
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
 
         if inputs_embeds is None:
             inputs_embeds = self.word_embeddings(input_ids)
@@ -655,11 +655,11 @@ class BertModel(BertPreTrainedModel):
         device = input_ids.device if input_ids is not None else inputs_embeds.device
 
         if attention_mask is None:
-            attention_mask = torch.ones(input_shape)
+            attention_mask = torch.ones(input_shape, device=device)
         if encoder_attention_mask is None:
-            encoder_attention_mask = torch.ones(input_shape)
+            encoder_attention_mask = torch.ones(input_shape, device=device)
         if token_type_ids is None:
-            token_type_ids = torch.zeros(input_shape, dtype=torch.long)
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
 
         # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
         # ourselves in which case we just need to make it broadcastable to all heads.

From 2f3a4210185f5311f6cfab3c91b30616c9a30fc8 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Wed, 6 Nov 2019 16:43:09 +0000
Subject: [PATCH 140/269] Fix other PyTorch models

---
 templates/adding_a_new_model/modeling_xxx.py | 6 ++++--
 transformers/modeling_distilbert.py          | 4 +++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
index d023f565f5..1f98c6406f 100644
--- a/templates/adding_a_new_model/modeling_xxx.py
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -309,10 +309,12 @@ class XxxModel(XxxPreTrainedModel):
         else:
             raise ValueError("You have to specify either input_ids or inputs_embeds")
 
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
         if attention_mask is None:
-            attention_mask = torch.ones(input_shape)
+            attention_mask = torch.ones(input_shape, device=device)
         if token_type_ids is None:
-            token_type_ids = torch.zeros(input_shape, dtype=torch.long)
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
 
         # We create a 3D attention mask from a 2D tensor mask.
         # Sizes are [batch_size, 1, 1, to_seq_length]
diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py
index aca1670852..00106627a8 100644
--- a/transformers/modeling_distilbert.py
+++ b/transformers/modeling_distilbert.py
@@ -450,8 +450,10 @@ class DistilBertModel(DistilBertPreTrainedModel):
         else:
             raise ValueError("You have to specify either input_ids or inputs_embeds")
 
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
         if attention_mask is None:
-            attention_mask = torch.ones(input_shape) # (bs, seq_length)
+            attention_mask = torch.ones(input_shape, device=device) # (bs, seq_length)
 
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head

From 1c542df7e554a2014051dd09becf60f157fed524 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Wed, 6 Nov 2019 16:26:31 -0500
Subject: [PATCH 141/269] Add RoBERTa-based GPT-2 Output Detector from OpenAI

converted from https://github.com/openai/gpt-2-output-dataset/tree/master/detector

Co-Authored-By: Lysandre Debut <lysandre.debut@reseau.eseo.fr>
Co-Authored-By: Jong Wook Kim <jongwook@nyu.edu>
Co-Authored-By: Jeff Wu <wuthefwasthat@gmail.com>
---
 docs/source/pretrained_models.rst     | 8 ++++++++
 transformers/configuration_roberta.py | 2 ++
 transformers/modeling_roberta.py      | 2 ++
 transformers/tokenization_roberta.py  | 6 ++++++
 4 files changed, 18 insertions(+)

diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index 559c81cbb0..edb47e7f1c 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -127,6 +127,14 @@ Here is the full list of the currently provided pretrained models together with
 |                   | ``roberta-large-mnli``                                     | | 24-layer, 1024-hidden, 16-heads, 355M parameters                                                                                    |
 |                   |                                                            | | ``roberta-large`` fine-tuned on `MNLI <http://www.nyu.edu/projects/bowman/multinli/>`__.                                            |
 |                   |                                                            | (see `details <https://github.com/pytorch/fairseq/tree/master/examples/roberta>`__)                                                   |
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``roberta-base-openai-detector``                           | | 12-layer, 768-hidden, 12-heads, 125M parameters                                                                                     |
+|                   |                                                            | | ``roberta-base`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model.                                             |
+|                   |                                                            | (see `details <https://github.com/openai/gpt-2-output-dataset/tree/master/detector>`__)                                               |
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``roberta-large-openai-detector``                          | | 24-layer, 1024-hidden, 16-heads, 355M parameters                                                                                    |
+|                   |                                                            | | ``roberta-large`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model.                                            |
+|                   |                                                            | (see `details <https://github.com/openai/gpt-2-output-dataset/tree/master/detector>`__)                                               |
 +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 | DistilBERT        | ``distilbert-base-uncased``                                | | 6-layer, 768-hidden, 12-heads, 66M parameters                                                                                       |
 |                   |                                                            | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint                                                   |
diff --git a/transformers/configuration_roberta.py b/transformers/configuration_roberta.py
index 367a85211d..842edac56e 100644
--- a/transformers/configuration_roberta.py
+++ b/transformers/configuration_roberta.py
@@ -29,6 +29,8 @@ ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-config.json",
     'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-config.json",
     'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-config.json",
+    'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-config.json",
+    'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-config.json",
 }
 
 
diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index 58b86000bb..fc27353d37 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -35,6 +35,8 @@ ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = {
     'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-pytorch_model.bin",
     'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-pytorch_model.bin",
     'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-pytorch_model.bin",
+    'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-pytorch_model.bin",
+    'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-pytorch_model.bin",
 }
 
 class RobertaEmbeddings(BertEmbeddings):
diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py
index 5e1300fa4d..df3e12bc7c 100644
--- a/transformers/tokenization_roberta.py
+++ b/transformers/tokenization_roberta.py
@@ -47,6 +47,8 @@ PRETRAINED_VOCAB_FILES_MAP = {
         'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json",
         'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-vocab.json",
         'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-vocab.json",
+        'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
+        'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json",
     },
     'merges_file':
     {
@@ -54,6 +56,8 @@ PRETRAINED_VOCAB_FILES_MAP = {
         'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt",
         'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-merges.txt",
         'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-merges.txt",
+        'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt",
+        'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt",
     },
 }
 
@@ -62,6 +66,8 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
     'roberta-large': 512,
     'roberta-large-mnli': 512,
     'distilroberta-base': 512,
+    'roberta-base-openai-detector': 512,
+    'roberta-large-openai-detector': 512,
 }
 
 

From 7a9aae1044aa4699310a8004f631fc0a4bdf1b65 Mon Sep 17 00:00:00 2001
From: Adrian Bauer <agbauer@gmail.com>
Date: Thu, 7 Nov 2019 17:08:39 -0500
Subject: [PATCH 142/269] Fix run_bertology.py

Make imports and args.overwrite_cache match run_glue.py
---
 examples/run_bertology.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/examples/run_bertology.py b/examples/run_bertology.py
index aae2d150d3..d1d05a1073 100644
--- a/examples/run_bertology.py
+++ b/examples/run_bertology.py
@@ -39,8 +39,9 @@ from transformers import (WEIGHTS_NAME,
 
 from run_glue import set_seed, load_and_cache_examples, ALL_MODELS, MODEL_CLASSES
 
-from utils_glue import (compute_metrics, convert_examples_to_features,
-                        output_modes, processors)
+from transformers import glue_compute_metrics as compute_metrics
+from transformers import glue_output_modes as output_modes
+from transformers import glue_processors as processors
 
 logger = logging.getLogger(__name__)
 
@@ -233,6 +234,8 @@ def main():
                         help="If > 0: limit the data to a subset of data_subset instances.")
     parser.add_argument("--overwrite_output_dir", action='store_true',
                         help="Whether to overwrite data in output directory")
+    parser.add_argument('--overwrite_cache', action='store_true',
+                        help="Overwrite the cached training and evaluation sets")
 
     parser.add_argument("--dont_normalize_importance_by_layer", action='store_true',
                         help="Don't normalize importance score by layers")

From b5d330d11820f4ac2cc8c909b1a6a77e0cd961e0 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Mon, 11 Nov 2019 10:15:14 -0500
Subject: [PATCH 143/269] Fix #1784

---
 templates/adding_a_new_model/tokenization_xxx.py | 2 +-
 transformers/tokenization_bert.py                | 2 +-
 transformers/tokenization_roberta.py             | 2 +-
 transformers/tokenization_utils.py               | 2 +-
 transformers/tokenization_xlm.py                 | 2 +-
 transformers/tokenization_xlnet.py               | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/templates/adding_a_new_model/tokenization_xxx.py b/templates/adding_a_new_model/tokenization_xxx.py
index 1b1325aab5..3d6b4ad9df 100644
--- a/templates/adding_a_new_model/tokenization_xxx.py
+++ b/templates/adding_a_new_model/tokenization_xxx.py
@@ -172,7 +172,7 @@ class XxxTokenizer(PreTrainedTokenizer):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
 
         if already_has_special_tokens:
diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py
index 8affdd9036..ded5072e58 100644
--- a/transformers/tokenization_bert.py
+++ b/transformers/tokenization_bert.py
@@ -220,7 +220,7 @@ class BertTokenizer(PreTrainedTokenizer):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
 
         if already_has_special_tokens:
diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py
index df3e12bc7c..b44e004997 100644
--- a/transformers/tokenization_roberta.py
+++ b/transformers/tokenization_roberta.py
@@ -120,7 +120,7 @@ class RobertaTokenizer(GPT2Tokenizer):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
         if already_has_special_tokens:
             if token_ids_1 is not None:
diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py
index ac765165e2..cd14cc4582 100644
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -951,7 +951,7 @@ class PreTrainedTokenizer(object):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
         return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))
 
diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py
index d09ce6b9dc..01f8721d98 100644
--- a/transformers/tokenization_xlm.py
+++ b/transformers/tokenization_xlm.py
@@ -781,7 +781,7 @@ class XLMTokenizer(PreTrainedTokenizer):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
 
         if already_has_special_tokens:
diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py
index deae8de336..a4f1a6e3ba 100644
--- a/transformers/tokenization_xlnet.py
+++ b/transformers/tokenization_xlnet.py
@@ -208,7 +208,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
 
         if already_has_special_tokens:

From 1c7253cc5f233978ddf6804efc71c0358acbac45 Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 11 Nov 2019 16:18:16 +0100
Subject: [PATCH 144/269] modeling: add DistilBertForTokenClassification
 implementation

---
 transformers/modeling_distilbert.py | 73 +++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py
index 00106627a8..d30f493c69 100644
--- a/transformers/modeling_distilbert.py
+++ b/transformers/modeling_distilbert.py
@@ -30,6 +30,7 @@ import numpy as np
 
 import torch
 import torch.nn as nn
+from torch.nn import CrossEntropyLoss
 
 from .modeling_utils import PreTrainedModel, prune_linear_layer
 from .configuration_distilbert import DistilBertConfig
@@ -702,3 +703,75 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel):
             outputs = (total_loss,) + outputs
 
         return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""DistilBert Model with a token classification head on top (a linear layer on top of
+                      the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+                      DISTILBERT_START_DOCSTRING,
+                      DISTILBERT_INPUTS_DOCSTRING)
+class DistilBertForTokenClassification(DistilBertPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the token classification loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification loss.
+        **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)``
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+        model = DistilBertForTokenClassification.from_pretrained('distilbert-base-uncased')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, scores = outputs[:2]
+
+    """
+    def __init__(self, config):
+        super(DistilBertForTokenClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.distilbert = DistilBertModel(config)
+        self.dropout = nn.Dropout(config.dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids=None, attention_mask=None, head_mask=None,
+                inputs_embeds=None, labels=None):
+
+        outputs = self.distilbert(input_ids,
+                            attention_mask=attention_mask,
+                            head_mask=head_mask,
+                            inputs_embeds=inputs_embeds)
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output)
+        logits = self.classifier(sequence_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+            # Only keep active parts of the loss
+            if attention_mask is not None:
+                active_loss = attention_mask.view(-1) == 1
+                active_logits = logits.view(-1, self.num_labels)[active_loss]
+                active_labels = labels.view(-1)[active_loss]
+                loss = loss_fct(active_logits, active_labels)
+            else:
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), scores, (hidden_states), (attentions)

From 1806eabf59fee4a8a79a3a80a927cbb3d0fbde45 Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 11 Nov 2019 16:18:48 +0100
Subject: [PATCH 145/269] module: add DistilBertForTokenClassification import

---
 transformers/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 53f3c39dc7..4f13122f70 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -93,6 +93,7 @@ if is_torch_available():
                                 ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel,
                                 DistilBertForSequenceClassification, DistilBertForQuestionAnswering,
+                                DistilBertForTokenClassification,
                                 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 

From 2b07b9e5ee14ac37fcef7bac958963d869b3b79a Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 11 Nov 2019 16:19:34 +0100
Subject: [PATCH 146/269] examples: add DistilBert support for NER fine-tuning

---
 examples/run_ner.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index b35d8298fe..1c5774df97 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -36,16 +36,18 @@ from utils_ner import convert_examples_to_features, get_labels, read_examples_fr
 from transformers import AdamW, WarmupLinearSchedule
 from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer
 from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer
+from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer
 
 logger = logging.getLogger(__name__)
 
 ALL_MODELS = sum(
-    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig)),
+    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)),
     ())
 
 MODEL_CLASSES = {
     "bert": (BertConfig, BertForTokenClassification, BertTokenizer),
-    "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer)
+    "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer),
+    "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer)
 }
 
 
@@ -121,9 +123,10 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id):
             batch = tuple(t.to(args.device) for t in batch)
             inputs = {"input_ids": batch[0],
                       "attention_mask": batch[1],
-                      "token_type_ids": batch[2] if args.model_type in ["bert", "xlnet"] else None,
-                      # XLM and RoBERTa don"t use segment_ids
                       "labels": batch[3]}
+            if args.model_type != "distilbert":
+                inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None  # XLM and RoBERTa don"t use segment_ids
+
             outputs = model(**inputs)
             loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)
 
@@ -206,9 +209,9 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=""
         with torch.no_grad():
             inputs = {"input_ids": batch[0],
                       "attention_mask": batch[1],
-                      "token_type_ids": batch[2] if args.model_type in ["bert", "xlnet"] else None,
-                      # XLM and RoBERTa don"t use segment_ids
                       "labels": batch[3]}
+            if args.model_type != "distilbert":
+                inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None  # XLM and RoBERTa don"t use segment_ids
             outputs = model(**inputs)
             tmp_eval_loss, logits = outputs[:2]
 
@@ -520,3 +523,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+

From 94e55253aef2ccb4b0de95e4aadd6432e3e6a65a Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 11 Nov 2019 16:20:15 +0100
Subject: [PATCH 147/269] tests: add test case for
 DistilBertForTokenClassification implementation

---
 .../tests/modeling_distilbert_test.py         | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py
index 937d03396d..8099c03586 100644
--- a/transformers/tests/modeling_distilbert_test.py
+++ b/transformers/tests/modeling_distilbert_test.py
@@ -23,6 +23,7 @@ from transformers import is_torch_available
 
 if is_torch_available():
     from transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM,
+                                    DistilBertForTokenClassification,
                                     DistilBertForQuestionAnswering, DistilBertForSequenceClassification)
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -180,6 +181,21 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
+        def create_and_check_distilbert_for_token_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = DistilBertForTokenClassification(config=config)
+            model.eval()
+
+            loss, logits = model(input_ids, attention_mask=input_mask, labels=token_labels)
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.seq_length, self.num_labels])
+            self.check_loss_output(result)
+
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
             (config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs
@@ -209,6 +225,10 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs)
 
+    def test_for_token_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_distilbert_for_token_classification(*config_and_inputs)
+
     # @pytest.mark.slow
     # def test_model_from_pretrained(self):
     #     cache_dir = "/tmp/transformers_test/"

From 8aba81a0b64bbf7a2dcd13eaceb543c5f38fd82f Mon Sep 17 00:00:00 2001
From: thomwolf <thomwolf@gmail.com>
Date: Tue, 12 Nov 2019 08:52:43 +0100
Subject: [PATCH 148/269] fix #1789

---
 README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 40b08583b1..17dfea6374 100644
--- a/README.md
+++ b/README.md
@@ -171,8 +171,7 @@ for model_class, tokenizer_class, pretrained_weights in MODELS:
 
 # Each architecture is provided with several class for fine-tuning on down-stream tasks, e.g.
 BERT_MODEL_CLASSES = [BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction,
-                      BertForSequenceClassification, BertForMultipleChoice, BertForTokenClassification,
-                      BertForQuestionAnswering]
+                      BertForSequenceClassification, BertForTokenClassification, BertForQuestionAnswering]
 
 # All the classes for an architecture can be initiated from pretrained weights for this architecture
 # Note that additional weights added for fine-tuning are only initialized

From 2e31176557d381171d44b5b51b72411b4c2e0601 Mon Sep 17 00:00:00 2001
From: ronakice <ronakice@hotmail.com>
Date: Tue, 12 Nov 2019 05:55:11 -0500
Subject: [PATCH 149/269] fix multi-gpu eval

---
 examples/run_glue.py                     | 4 ++++
 examples/run_lm_finetuning.py            | 4 ++++
 examples/run_multiple_choice.py          | 4 ++++
 examples/run_ner.py                      | 4 ++++
 examples/run_squad.py                    | 4 ++++
 examples/run_summarization_finetuning.py | 4 ++++
 6 files changed, 24 insertions(+)

diff --git a/examples/run_glue.py b/examples/run_glue.py
index 1558a812c3..f82e589301 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -224,6 +224,10 @@ def evaluate(args, model, tokenizer, prefix=""):
         eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
+        # multi-gpu eval
+        if args.n_gpu > 1:
+            model = torch.nn.DataParallel(model)
+
         # Eval!
         logger.info("***** Running evaluation {} *****".format(prefix))
         logger.info("  Num examples = %d", len(eval_dataset))
diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 2044cfe9e8..d9ee2fdb2b 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -300,6 +300,10 @@ def evaluate(args, model, tokenizer, prefix=""):
     eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
     eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
+    # multi-gpu evaluate
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
     # Eval!
     logger.info("***** Running evaluation {} *****".format(prefix))
     logger.info("  Num examples = %d", len(eval_dataset))
diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py
index 638bbe74f1..c9e13e198d 100644
--- a/examples/run_multiple_choice.py
+++ b/examples/run_multiple_choice.py
@@ -229,6 +229,10 @@ def evaluate(args, model, tokenizer, prefix="", test=False):
         eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
+        # multi-gpu evaluate
+        if args.n_gpu > 1:
+            model = torch.nn.DataParallel(model)
+
         # Eval!
         logger.info("***** Running evaluation {} *****".format(prefix))
         logger.info("  Num examples = %d", len(eval_dataset))
diff --git a/examples/run_ner.py b/examples/run_ner.py
index b35d8298fe..c12709e37b 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -191,6 +191,10 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=""
     eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
     eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
+    # multi-gpu evaluate
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
     # Eval!
     logger.info("***** Running evaluation %s *****", prefix)
     logger.info("  Num examples = %d", len(eval_dataset))
diff --git a/examples/run_squad.py b/examples/run_squad.py
index d9dc2abfde..ad4656462d 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -217,6 +217,10 @@ def evaluate(args, model, tokenizer, prefix=""):
     eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset)
     eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
+    # multi-gpu evaluate
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
     # Eval!
     logger.info("***** Running evaluation {} *****".format(prefix))
     logger.info("  Num examples = %d", len(dataset))
diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py
index 448505c727..f5604c2669 100644
--- a/examples/run_summarization_finetuning.py
+++ b/examples/run_summarization_finetuning.py
@@ -275,6 +275,10 @@ def evaluate(args, model, tokenizer, prefix=""):
         eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size
     )
 
+    # multi-gpu evaluate
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+        
     logger.info("***** Running evaluation {} *****".format(prefix))
     logger.info("  Num examples = %d", len(eval_dataset))
     logger.info("  Batch size = %d", args.eval_batch_size)

From 7246d3c2f93c4461f3ec8ada7a26a002d8f196ea Mon Sep 17 00:00:00 2001
From: Michael Watkins <me@michaelwatkins.eu>
Date: Wed, 6 Nov 2019 13:18:16 +0200
Subject: [PATCH 150/269] Consider do_lower_case in PreTrainedTokenizer

As pointed out in #1545, when using an uncased model, and adding
a new uncased token, the tokenizer does not correctly identify this
in the case that the input text contains the token in a cased format.

For instance, if we load bert-base-uncased into BertTokenizer, and
then use .add_tokens() to add "cool-token", we get the expected
result for .tokenize('this is a cool-token'). However, we get a
possibly unexpected result for .tokenize('this is a cOOl-Token'),
which in fact mirrors the result for the former from before the new
token was added.

This commit adds
- functionality to PreTrainedTokenizer to handle this
situation in case a tokenizer (currently Bert, DistilBert,
and XLNet) has the do_lower_case=True kwarg by:
    1) lowercasing tokens added with .add_tokens()
    2) lowercasing text at the beginning of .tokenize()
- new common test case for tokenizers

https://github.com/huggingface/transformers/issues/1545
---
 .../tests/tokenization_tests_commons.py       | 31 ++++++++++++++++++-
 transformers/tokenization_utils.py            |  5 +++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py
index a921696b77..287e6fc7b3 100644
--- a/transformers/tests/tokenization_tests_commons.py
+++ b/transformers/tests/tokenization_tests_commons.py
@@ -110,6 +110,36 @@ class CommonTestCases:
 
             self.assertListEqual(subwords, subwords_loaded)
 
+        def test_added_tokens_do_lower_case(self):
+            tokenizer = self.get_tokenizer(do_lower_case=True)
+
+            text = "aaaaa bbbbbb low cccccccccdddddddd l"
+            text2 = "AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l"
+
+            toks0 = tokenizer.tokenize(text)  # toks before adding new_toks
+
+            new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", 'AAAAA BBBBBB', 'CCCCCCCCCDDDDDDDD']
+            added = tokenizer.add_tokens(new_toks)
+            self.assertEqual(added, 2)
+
+            toks = tokenizer.tokenize(text)
+            toks2 = tokenizer.tokenize(text2)
+
+            self.assertEqual(len(toks), len(toks2))
+            self.assertNotEqual(len(toks), len(toks0))  # toks0 should be longer
+            self.assertListEqual(toks, toks2)
+
+            tokenizer = self.get_tokenizer(do_lower_case=False)
+
+            added = tokenizer.add_tokens(new_toks)
+            self.assertEqual(added, 4)
+
+            toks = tokenizer.tokenize(text)
+            toks2 = tokenizer.tokenize(text2)
+
+            self.assertEqual(len(toks), len(toks2))  # Length should still be the same
+            self.assertNotEqual(len(toks), len(toks0))
+            self.assertNotEqual(toks[0], toks2[0])  # But at least the first tokens should differ
 
         def test_add_tokens_tokenizer(self):
             tokenizer = self.get_tokenizer()
@@ -160,7 +190,6 @@ class CommonTestCases:
             self.assertEqual(tokens[0], tokenizer.eos_token_id)
             self.assertEqual(tokens[-2], tokenizer.pad_token_id)
 
-
         def test_required_methods_tokenizer(self):
             tokenizer = self.get_tokenizer()
             input_text, output_text = self.get_input_output_texts()
diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py
index cd14cc4582..fc31c10d25 100644
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -512,6 +512,8 @@ class PreTrainedTokenizer(object):
         to_add_tokens = []
         for token in new_tokens:
             assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode))
+            if self.init_kwargs.get('do_lower_case', False):
+                token = token.lower()
             if token != self.unk_token and \
                     self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \
                     token not in to_add_tokens:
@@ -605,6 +607,9 @@ class PreTrainedTokenizer(object):
 
             Take care of added tokens.
         """
+        if self.init_kwargs.get('do_lower_case', False):
+            text = text.lower()
+
         def split_on_token(tok, text):
             result = []
             split_text = text.split(tok)

From d409aca32632718afbcd098de2bb11b9b71b7df1 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 12 Nov 2019 10:59:37 -0500
Subject: [PATCH 151/269] Clarify the use of past in GPT2 and CTRL

---
 transformers/modeling_ctrl.py |  9 ++++++---
 transformers/modeling_gpt2.py | 12 ++++++++----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py
index 405e33602a..1ed9e6ebb1 100644
--- a/transformers/modeling_ctrl.py
+++ b/transformers/modeling_ctrl.py
@@ -220,7 +220,8 @@ CTRL_INPUTS_DOCSTRING = r"""    Inputs:
         **past**:
             list of ``torch.FloatTensor`` (one for each layer):
             that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model
-            (see `past` output below). Can be used to speed up sequential decoding.
+            (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:
             Mask to avoid performing attention on padding token indices.
             Mask values selected in ``[0, 1]``:
@@ -252,7 +253,8 @@ class CTRLModel(CTRLPreTrainedModel):
         **past**:
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             that contains pre-computed hidden-states (key and values in the attention blocks).
-            Can be used (see `past` input) to speed up sequential decoding.
+            Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
             list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
             of shape ``(batch_size, sequence_length, hidden_size)``:
@@ -437,7 +439,8 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
         **past**:
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             that contains pre-computed hidden-states (key and values in the attention blocks).
-            Can be used (see `past` input) to speed up sequential decoding.
+            Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
             list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
             of shape ``(batch_size, sequence_length, hidden_size)``:
diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py
index e3d26797c8..35bc5c8d6e 100644
--- a/transformers/modeling_gpt2.py
+++ b/transformers/modeling_gpt2.py
@@ -298,7 +298,8 @@ GPT2_INPUTS_DOCSTRING = r"""    Inputs:
         **past**:
             list of ``torch.FloatTensor`` (one for each layer):
             that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model
-            (see `past` output below). Can be used to speed up sequential decoding.
+            (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:
             Mask to avoid performing attention on padding token indices.
             Mask values selected in ``[0, 1]``:
@@ -330,7 +331,8 @@ class GPT2Model(GPT2PreTrainedModel):
         **past**:
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             that contains pre-computed hidden-states (key and values in the attention blocks).
-            Can be used (see `past` input) to speed up sequential decoding.
+            Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
             list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
             of shape ``(batch_size, sequence_length, hidden_size)``:
@@ -503,7 +505,8 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
         **past**:
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             that contains pre-computed hidden-states (key and values in the attention blocks).
-            Can be used (see `past` input) to speed up sequential decoding.
+            Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
             list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
             of shape ``(batch_size, sequence_length, hidden_size)``:
@@ -595,7 +598,8 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
         **past**:
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             that contains pre-computed hidden-states (key and values in the attention blocks).
-            Can be used (see `past` input) to speed up sequential decoding.
+            Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model 
+            should not be passed as input ids as they have already been computed.
         **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
             list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
             of shape ``(batch_size, sequence_length, hidden_size)``:

From dd6b2e05e17d7f3eb1533e9df47f7fd8a1aafb0f Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 15:18:09 -0500
Subject: [PATCH 152/269] whitespace

---
 .gitignore                        |  3 ++-
 transformers/modeling_tf_utils.py | 14 +++++++-------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index ec60c045c4..c789666707 100644
--- a/.gitignore
+++ b/.gitignore
@@ -137,4 +137,5 @@ examples/runs
 serialization_dir
 
 # emacs
-*.*~
\ No newline at end of file
+*.*~
+debug.env
diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py
index f626327283..446fcad131 100644
--- a/transformers/modeling_tf_utils.py
+++ b/transformers/modeling_tf_utils.py
@@ -477,10 +477,10 @@ def shape_list(x):
     return [dynamic[i] if s is None else s for i, s in enumerate(static)]
 
 def get_initializer(initializer_range=0.02):
-  """Creates a `tf.initializers.truncated_normal` with the given range.
-  Args:
-    initializer_range: float, initializer range for stddev.
-  Returns:
-    TruncatedNormal initializer with stddev = `initializer_range`.
-  """
-  return tf.keras.initializers.TruncatedNormal(stddev=initializer_range)
+    """Creates a `tf.initializers.truncated_normal` with the given range.
+    Args:
+        initializer_range: float, initializer range for stddev.
+    Returns:
+        TruncatedNormal initializer with stddev = `initializer_range`.
+    """
+    return tf.keras.initializers.TruncatedNormal(stddev=initializer_range)

From 872403be1cd62c9576e0938e2531749fbb6733ea Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 15:22:54 -0500
Subject: [PATCH 153/269] This is not a @property after all

---
 templates/adding_a_new_model/modeling_xxx.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py
index 1f98c6406f..94c4b0db9a 100644
--- a/templates/adding_a_new_model/modeling_xxx.py
+++ b/templates/adding_a_new_model/modeling_xxx.py
@@ -284,7 +284,6 @@ class XxxModel(XxxPreTrainedModel):
 
         self.init_weights()
 
-    @property
     def get_input_embeddings(self):
         return self.embeddings.word_embeddings
 

From 70d97ddd607f3d45e6f7af3ea2f60b3cc7cd13d5 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 16:30:22 -0500
Subject: [PATCH 154/269] [TF models] Common attributes as per #1721

---
 transformers/modeling_tf_bert.py              |  9 +++++++++
 transformers/modeling_tf_ctrl.py              |  6 ++++++
 transformers/modeling_tf_distilbert.py        |  6 ++++++
 transformers/modeling_tf_gpt2.py              |  9 +++++++++
 transformers/modeling_tf_openai.py            |  9 +++++++++
 transformers/modeling_tf_roberta.py           |  6 ++++++
 transformers/modeling_tf_transfo_xl.py        |  3 +++
 transformers/modeling_tf_utils.py             | 15 +++++++++++++++
 transformers/modeling_tf_xlm.py               |  5 +++++
 transformers/modeling_tf_xlnet.py             |  6 ++++++
 transformers/tests/modeling_tf_common_test.py | 10 ++++++++++
 11 files changed, 84 insertions(+)

diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py
index a1275db974..66d5efd87c 100644
--- a/transformers/modeling_tf_bert.py
+++ b/transformers/modeling_tf_bert.py
@@ -460,6 +460,9 @@ class TFBertMainLayer(tf.keras.layers.Layer):
         self.encoder = TFBertEncoder(config, name='encoder')
         self.pooler = TFBertPooler(config, name='pooler')
 
+    def get_input_embeddings(self):
+        return self.embeddings
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -702,6 +705,9 @@ class TFBertForPreTraining(TFBertPreTrainedModel):
         self.nsp = TFBertNSPHead(config, name='nsp___cls')
         self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls')
 
+    def get_output_embeddings(self):
+        return self.bert.embeddings
+
     def call(self, inputs, **kwargs):
         outputs = self.bert(inputs, **kwargs)
 
@@ -747,6 +753,9 @@ class TFBertForMaskedLM(TFBertPreTrainedModel):
         self.bert = TFBertMainLayer(config, name='bert')
         self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls')
 
+    def get_output_embeddings(self):
+        return self.bert.embeddings
+
     def call(self, inputs, **kwargs):
         outputs = self.bert(inputs, **kwargs)
 
diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py
index dea590c5c5..99738a8b14 100644
--- a/transformers/modeling_tf_ctrl.py
+++ b/transformers/modeling_tf_ctrl.py
@@ -192,6 +192,9 @@ class TFCTRLMainLayer(tf.keras.layers.Layer):
                                  name='h_._{}'.format(i)) for i in range(config.n_layer)]
         self.layernorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name="layernorm")
 
+    def get_input_embeddings(self):
+        return self.w
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -480,6 +483,9 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel):
 
         self.lm_head = TFCTRLLMHead(config, self.transformer.w, name="lm_head")
 
+    def get_output_embeddings(self):
+        return self.lm_head.input_embeddings
+
     def call(self, inputs, **kwargs):
         transformer_outputs = self.transformer(inputs, **kwargs)
         hidden_states = transformer_outputs[0]
diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py
index 65acb9e142..4b1f3e676b 100644
--- a/transformers/modeling_tf_distilbert.py
+++ b/transformers/modeling_tf_distilbert.py
@@ -398,6 +398,9 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer):
         self.embeddings = TFEmbeddings(config, name="embeddings")   # Embeddings
         self.transformer = TFTransformer(config, name="transformer") # Encoder
 
+    def get_input_embeddings(self):
+        return self.embeddings
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -613,6 +616,9 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel):
         self.vocab_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="vocab_layer_norm")
         self.vocab_projector = TFDistilBertLMHead(config, self.distilbert.embeddings, name="vocab_projector")
 
+    def get_output_embeddings(self):
+        return self.vocab_projector.input_embeddings
+
     def call(self, inputs, **kwargs):
         distilbert_output = self.distilbert(inputs, **kwargs)
 
diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py
index 50d58a6749..23866a1a0a 100644
--- a/transformers/modeling_tf_gpt2.py
+++ b/transformers/modeling_tf_gpt2.py
@@ -219,6 +219,9 @@ class TFGPT2MainLayer(tf.keras.layers.Layer):
                           name='h_._{}'.format(i)) for i in range(config.n_layer)]
         self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f')
 
+    def get_input_embeddings(self):
+        return self.wte
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -490,6 +493,9 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel):
         super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs)
         self.transformer = TFGPT2MainLayer(config, name='transformer')
 
+    def get_output_embeddings(self):
+        return self.transformer.wte
+
     def call(self, inputs, **kwargs):
         transformer_outputs = self.transformer(inputs, **kwargs)
         hidden_states = transformer_outputs[0]
@@ -560,6 +566,9 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
         self.transformer = TFGPT2MainLayer(config, name='transformer')
         self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head')
 
+    def get_output_embeddings(self):
+        return self.transformer.wte
+
     def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py
index 18afa85dce..bddd9338b1 100644
--- a/transformers/modeling_tf_openai.py
+++ b/transformers/modeling_tf_openai.py
@@ -217,6 +217,9 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer):
                           scale=True,
                           name='h_._{}'.format(i)) for i in range(config.n_layer)]
 
+    def get_input_embeddings(self):
+        return self.tokens_embed
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -462,6 +465,9 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel):
         super(TFOpenAIGPTLMHeadModel, self).__init__(config, *inputs, **kwargs)
         self.transformer = TFOpenAIGPTMainLayer(config, name='transformer')
 
+    def get_output_embeddings(self):
+        return self.transformer.tokens_embed
+
     def call(self, inputs, **kwargs):
         transformer_outputs = self.transformer(inputs, **kwargs)
         hidden_states = transformer_outputs[0]
@@ -524,6 +530,9 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel):
         self.transformer = TFOpenAIGPTMainLayer(config, name='transformer')
         self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head')
 
+    def get_output_embeddings(self):
+        return self.transformer.tokens_embed
+
     def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py
index 32abea659e..c335910dc6 100644
--- a/transformers/modeling_tf_roberta.py
+++ b/transformers/modeling_tf_roberta.py
@@ -65,6 +65,9 @@ class TFRobertaMainLayer(TFBertMainLayer):
         super(TFRobertaMainLayer, self).__init__(config, **kwargs)
         self.embeddings = TFRobertaEmbeddings(config, name='embeddings')
 
+    def get_input_embeddings(self):
+        return self.embeddings
+
 
 class TFRobertaPreTrainedModel(TFPreTrainedModel):
     """ An abstract class to handle weights initialization and
@@ -280,6 +283,9 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel):
         self.roberta = TFRobertaMainLayer(config, name="roberta")
         self.lm_head = TFRobertaLMHead(config, self.roberta.embeddings, name="lm_head")
 
+    def get_output_embeddings(self):
+        return self.lm_head.decoder
+
     def call(self, inputs, **kwargs):
         outputs = self.roberta(inputs, **kwargs)
 
diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py
index ec37aedd74..8c2a35b352 100644
--- a/transformers/modeling_tf_transfo_xl.py
+++ b/transformers/modeling_tf_transfo_xl.py
@@ -413,6 +413,9 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer):
                                             name='r_r_bias')
         super(TFTransfoXLMainLayer, self).build(input_shape)
 
+    def get_input_embeddings(self):
+        return self.word_emb
+
     def _resize_token_embeddings(self, new_num_tokens):
         return self.word_emb
 
diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py
index 446fcad131..e08605d154 100644
--- a/transformers/modeling_tf_utils.py
+++ b/transformers/modeling_tf_utils.py
@@ -65,6 +65,21 @@ class TFPreTrainedModel(tf.keras.Model):
         # Save config in model
         self.config = config
 
+    def get_input_embeddings(self):
+        """ Get model's input embeddings
+        """
+        base_model = getattr(self, self.base_model_prefix, self)
+        if base_model is not self:
+            return base_model.get_input_embeddings()
+        else:
+            raise NotImplementedError
+
+    def get_output_embeddings(self):
+        """ Get model's output embeddings
+            Return None if the model doesn't have output embeddings
+        """
+        return None  # Overwrite for models with output embeddings
+
     def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None):
         """ Build a resized Embedding Variable from a provided token Embedding Module.
             Increasing the size will add newly initialized vectors at the end
diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py
index 496d7d72a8..20fbdca732 100644
--- a/transformers/modeling_tf_xlm.py
+++ b/transformers/modeling_tf_xlm.py
@@ -277,6 +277,9 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
                     self.prune_heads({int(layer): list(map(int, heads))})
 
 
+    def get_input_embeddings(self):
+        return self.embeddings
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -641,6 +644,8 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel):
         self.transformer = TFXLMMainLayer(config, name='transformer')
         self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer_._proj')
 
+    def get_output_embeddings(self):
+        return self.pred_layer.input_embeddings
 
     def call(self, inputs, **kwargs):
         transformer_outputs = self.transformer(inputs, **kwargs)
diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py
index bb33e45790..7ab95e7c9f 100644
--- a/transformers/modeling_tf_xlnet.py
+++ b/transformers/modeling_tf_xlnet.py
@@ -371,6 +371,9 @@ class TFXLNetMainLayer(tf.keras.layers.Layer):
         self.layer = [TFXLNetLayer(config, name='layer_._{}'.format(i)) for i in range(config.n_layer)]
         self.dropout = tf.keras.layers.Dropout(config.dropout)
 
+    def get_input_embeddings(self):
+        return self.word_embedding
+
     def build(self, input_shape):
         initializer = get_initializer(self.initializer_range)
         self.mask_emb = self.add_weight(shape=(1, 1, self.d_model),
@@ -854,6 +857,9 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel):
         self.transformer = TFXLNetMainLayer(config, name='transformer')
         self.lm_loss = TFXLNetLMHead(config, self.transformer.word_embedding, name='lm_loss')
 
+    def get_output_embeddings(self):
+        return self.lm_loss.input_embeddings
+
     def call(self, inputs, **kwargs):
         transformer_outputs = self.transformer(inputs, **kwargs)
         hidden_state = transformer_outputs[0]
diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py
index f636c42889..0be5fe8e9c 100644
--- a/transformers/tests/modeling_tf_common_test.py
+++ b/transformers/tests/modeling_tf_common_test.py
@@ -360,6 +360,16 @@ class TFCommonTestCases:
             #     self.assertTrue(models_equal)
 
 
+        def test_model_common_attributes(self):
+            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+            for model_class in self.all_model_classes:
+                model = model_class(config)
+                assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer)
+                x = model.get_output_embeddings()
+                assert x is None or instanceof(x, tf.keras.layers.Layer)
+
+
         def test_tie_model_weights(self):
             pass
             # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

From 9d2398fd99d758126709109e5830c6ffe95606e7 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 16:37:08 -0500
Subject: [PATCH 155/269] Ooopsie

---
 transformers/tests/modeling_tf_common_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py
index 0be5fe8e9c..ca7beb0925 100644
--- a/transformers/tests/modeling_tf_common_test.py
+++ b/transformers/tests/modeling_tf_common_test.py
@@ -367,7 +367,7 @@ class TFCommonTestCases:
                 model = model_class(config)
                 assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer)
                 x = model.get_output_embeddings()
-                assert x is None or instanceof(x, tf.keras.layers.Layer)
+                assert x is None or isinstance(x, tf.keras.layers.Layer)
 
 
         def test_tie_model_weights(self):

From 2f17464266ef5fe8314f78de1320e16cdf29d909 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 19:56:45 -0500
Subject: [PATCH 156/269] [common attributes] Slightly sharper test coverage

---
 transformers/tests/modeling_common_test.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 38b2ceafa4..777e62459b 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -468,9 +468,15 @@ class CommonTestCases:
 
             for model_class in self.all_model_classes:
                 model = model_class(config)
-                model.get_input_embeddings()
+                self.assertIsInstance(
+                    model.get_input_embeddings(),
+                    torch.nn.Embedding
+                )
                 model.set_input_embeddings(torch.nn.Embedding(10, 10))
-                model.get_output_embeddings()
+                x = model.get_output_embeddings()
+                self.assertTrue(
+                    x is None or isinstance(x, torch.nn.Linear)
+                )
 
         def test_tie_model_weights(self):
             if not self.test_torchscript:

From 2aef2f0bbcd3b192af18718684615019a7777a9b Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 20:03:19 -0500
Subject: [PATCH 157/269] [common attributes] Fix previous commit for
 transfo-xl

---
 transformers/__init__.py                   | 1 +
 transformers/tests/modeling_common_test.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 53f3c39dc7..d922f52a1d 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -72,6 +72,7 @@ if is_torch_available():
                                 OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel,
                                 load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel,
+                                    AdaptiveEmbedding,
                                     load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model,
                                 GPT2LMHeadModel, GPT2DoubleHeadsModel,
diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py
index 777e62459b..baf1531403 100644
--- a/transformers/tests/modeling_common_test.py
+++ b/transformers/tests/modeling_common_test.py
@@ -35,7 +35,7 @@ if is_torch_available():
     import torch
     import numpy as np
 
-    from transformers import (PretrainedConfig, PreTrainedModel,
+    from transformers import (AdaptiveEmbedding, PretrainedConfig, PreTrainedModel,
                                     BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP,
                                     GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP)
 else:
@@ -470,7 +470,7 @@ class CommonTestCases:
                 model = model_class(config)
                 self.assertIsInstance(
                     model.get_input_embeddings(),
-                    torch.nn.Embedding
+                    (torch.nn.Embedding, AdaptiveEmbedding)
                 )
                 model.set_input_embeddings(torch.nn.Embedding(10, 10))
                 x = model.get_output_embeddings()

From 155c782a2ccd103cf63ad48a2becd7c76a7d2115 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 11 Nov 2019 22:19:14 -0500
Subject: [PATCH 158/269] [inputs_embeds] All TF models + tests

---
 transformers/modeling_tf_bert.py              | 59 +++++++++++++------
 transformers/modeling_tf_ctrl.py              | 27 ++++++---
 transformers/modeling_tf_distilbert.py        | 38 ++++++++----
 transformers/modeling_tf_gpt2.py              | 44 +++++++++-----
 transformers/modeling_tf_openai.py            | 44 +++++++++-----
 transformers/modeling_tf_roberta.py           | 10 +++-
 transformers/modeling_tf_transfo_xl.py        | 53 +++++++++++------
 transformers/modeling_tf_xlm.py               | 31 +++++++---
 transformers/modeling_tf_xlnet.py             | 26 ++++++--
 transformers/tests/modeling_tf_bert_test.py   |  4 --
 transformers/tests/modeling_tf_common_test.py | 21 +++++++
 11 files changed, 252 insertions(+), 105 deletions(-)

diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py
index 66d5efd87c..ad0815e2ca 100644
--- a/transformers/modeling_tf_bert.py
+++ b/transformers/modeling_tf_bert.py
@@ -142,19 +142,25 @@ class TFBertEmbeddings(tf.keras.layers.Layer):
 
     def _embedding(self, inputs, training=False):
         """Applies embedding based on inputs tensor."""
-        input_ids, position_ids, token_type_ids = inputs
+        input_ids, position_ids, token_type_ids, inputs_embeds = inputs
 
-        seq_length = tf.shape(input_ids)[1]
+        if input_ids is not None:
+            input_shape = tf.shape(input_ids)
+        else:
+            input_shape = tf.shape(inputs_embeds)[:-1]
+        
+        seq_length = input_shape[1]
         if position_ids is None:
             position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :]
         if token_type_ids is None:
-            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+            token_type_ids = tf.fill(input_shape, 0)
 
-        words_embeddings = tf.gather(self.word_embeddings, input_ids)
+        if inputs_embeds is None:
+            inputs_embeds = tf.gather(self.word_embeddings, input_ids)
         position_embeddings = self.position_embeddings(position_ids)
         token_type_embeddings = self.token_type_embeddings(token_type_ids)
 
-        embeddings = words_embeddings + position_embeddings + token_type_embeddings
+        embeddings = inputs_embeds + position_embeddings + token_type_embeddings
         embeddings = self.LayerNorm(embeddings)
         embeddings = self.dropout(embeddings, training=training)
         return embeddings
@@ -473,28 +479,39 @@ class TFBertMainLayer(tf.keras.layers.Layer):
         """
         raise NotImplementedError
 
-    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
             token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
             position_ids = inputs[3] if len(inputs) > 3 else position_ids
             head_mask = inputs[4] if len(inputs) > 4 else head_mask
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds
+            assert len(inputs) <= 6, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 6, "Too many inputs."
         else:
             input_ids = inputs
 
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.shape
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.shape[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if attention_mask is None:
-            attention_mask = tf.fill(tf.shape(input_ids), 1)
+            attention_mask = tf.fill(input_shape, 1)
         if token_type_ids is None:
-            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+            token_type_ids = tf.fill(input_shape, 0)
 
         # We create a 3D attention mask from a 2D tensor mask.
         # Sizes are [batch_size, 1, 1, to_seq_length]
@@ -523,7 +540,7 @@ class TFBertMainLayer(tf.keras.layers.Layer):
             head_mask = [None] * self.num_hidden_layers
             # head_mask = tf.constant([0] * self.num_hidden_layers)
 
-        embedding_output = self.embeddings([input_ids, position_ids, token_type_ids], training=training)
+        embedding_output = self.embeddings([input_ids, position_ids, token_type_ids, inputs_embeds], training=training)
         encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training)
 
         sequence_output = encoder_outputs[0]
@@ -901,33 +918,39 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel):
                                                 kernel_initializer=get_initializer(config.initializer_range),
                                                 name='classifier')
 
-    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
             token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
             position_ids = inputs[3] if len(inputs) > 3 else position_ids
             head_mask = inputs[4] if len(inputs) > 4 else head_mask
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds
+            assert len(inputs) <= 6, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 6, "Too many inputs."
         else:
             input_ids = inputs
 
-        num_choices = tf.shape(input_ids)[1]
-        seq_length = tf.shape(input_ids)[2]
+        if input_ids is not None:
+            num_choices = tf.shape(input_ids)[1]
+            seq_length = tf.shape(input_ids)[2]
+        else:
+            num_choices = tf.shape(inputs_embeds)[1]
+            seq_length = tf.shape(inputs_embeds)[2]
 
-        flat_input_ids = tf.reshape(input_ids, (-1, seq_length))
+        flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None
         flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None
         flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None
         flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None
 
-        flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask]
+        flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask, inputs_embeds]
 
         outputs = self.bert(flat_inputs, training=training)
 
diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py
index 99738a8b14..ae66dbc82c 100644
--- a/transformers/modeling_tf_ctrl.py
+++ b/transformers/modeling_tf_ctrl.py
@@ -204,7 +204,7 @@ class TFCTRLMainLayer(tf.keras.layers.Layer):
         """
         raise NotImplementedError
 
-    def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+    def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             past = inputs[1] if len(inputs) > 1 else past
@@ -212,7 +212,8 @@ class TFCTRLMainLayer(tf.keras.layers.Layer):
             token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids
             position_ids = inputs[4] if len(inputs) > 4 else position_ids
             head_mask = inputs[5] if len(inputs) > 5 else head_mask
-            assert len(inputs) <= 6, "Too many inputs."
+            inputs_embeds = inputs[6] if len(inputs) > 6 else inputs_embeds
+            assert len(inputs) <= 7, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             past = inputs.get('past', past)
@@ -220,12 +221,20 @@ class TFCTRLMainLayer(tf.keras.layers.Layer):
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 6, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 7, "Too many inputs."
         else:
             input_ids = inputs
 
-        input_shape = shape_list(input_ids)
-        input_ids = tf.reshape(input_ids, [-1, input_shape[-1]])
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = shape_list(input_ids)
+            input_ids = tf.reshape(input_ids, [-1, input_shape[-1]])
+        elif inputs_embeds is not None:
+            input_shape = shape_list(inputs_embeds)[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
 
         if past is None:
             past_length = 0
@@ -233,8 +242,8 @@ class TFCTRLMainLayer(tf.keras.layers.Layer):
         else:
             past_length = shape_list(past[0][0])[-2]
         if position_ids is None:
-            position_ids = tf.range(past_length, shape_list(input_ids)[-1] + past_length, dtype=tf.int32)[tf.newaxis, :]
-            position_ids = tf.tile(position_ids, [shape_list(input_ids)[0], 1])
+            position_ids = tf.range(past_length, input_shape[-1] + past_length, dtype=tf.int32)[tf.newaxis, :]
+            position_ids = tf.tile(position_ids, [input_shape[0], 1])
 
         # Attention mask.
         if attention_mask is not None:
@@ -273,8 +282,8 @@ class TFCTRLMainLayer(tf.keras.layers.Layer):
             token_type_embeds = 0
         position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]])
 
-        inputs_embeds = self.w(input_ids, mode='embedding')
-        # x = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded
+        if inputs_embeds is None:
+            inputs_embeds = self.w(input_ids, mode='embedding')
         seq_len = input_shape[-1]
         mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
 
diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py
index 4b1f3e676b..6d393bb95d 100644
--- a/transformers/modeling_tf_distilbert.py
+++ b/transformers/modeling_tf_distilbert.py
@@ -96,7 +96,7 @@ class TFEmbeddings(tf.keras.layers.Layer):
                 initializer=get_initializer(self.initializer_range))
         super(TFEmbeddings, self).build(input_shape)
 
-    def call(self, inputs, mode="embedding", training=False):
+    def call(self, inputs, inputs_embeds=None, mode="embedding", training=False):
         """Get token embeddings of inputs.
         Args:
             inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids)
@@ -112,13 +112,13 @@ class TFEmbeddings(tf.keras.layers.Layer):
             https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
         if mode == "embedding":
-            return self._embedding(inputs, training=training)
+            return self._embedding(inputs, inputs_embeds=inputs_embeds, training=training)
         elif mode == "linear":
             return self._linear(inputs)
         else:
             raise ValueError("mode {} is not valid.".format(mode))
 
-    def _embedding(self, inputs, training=False):
+    def _embedding(self, inputs, inputs_embeds=None, training=False):
         """
         Parameters
         ----------
@@ -136,14 +136,19 @@ class TFEmbeddings(tf.keras.layers.Layer):
         else:
             input_ids, position_ids = inputs
 
-        seq_length = tf.shape(input_ids)[1]
+        if input_ids is not None:
+            seq_length = tf.shape(input_ids)[1]
+        else:
+            seq_length = tf.shape(inputs_embeds)[1]
+
         if position_ids is None:
             position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :]
 
-        word_embeddings = tf.gather(self.word_embeddings, input_ids)
+        if inputs_embeds is None:
+            inputs_embeds = tf.gather(self.word_embeddings, input_ids)
         position_embeddings = self.position_embeddings(position_ids)  # (bs, max_seq_length, dim)
 
-        embeddings = word_embeddings + position_embeddings            # (bs, max_seq_length, dim)
+        embeddings = inputs_embeds + position_embeddings              # (bs, max_seq_length, dim)
         embeddings = self.LayerNorm(embeddings)                       # (bs, max_seq_length, dim)
         embeddings = self.dropout(embeddings, training=training)      # (bs, max_seq_length, dim)
         return embeddings
@@ -407,22 +412,33 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer):
     def _prune_heads(self, heads_to_prune):
         raise NotImplementedError
 
-    def call(self, inputs, attention_mask=None, head_mask=None, training=False):
+    def call(self, inputs, attention_mask=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
             head_mask = inputs[2] if len(inputs) > 2 else head_mask
-            assert len(inputs) <= 3, "Too many inputs."
+            inputs_embeds = inputs[3] if len(inputs) > 3 else inputs_embeds
+            assert len(inputs) <= 4, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 3, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 4, "Too many inputs."
         else:
             input_ids = inputs
 
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = shape_list(input_ids)
+        elif inputs_embeds is not None:
+            input_shape = shape_list(inputs_embeds)[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if attention_mask is None:
-            attention_mask = tf.ones(shape_list(input_ids)) # (bs, seq_length)
+            attention_mask = tf.ones(input_shape) # (bs, seq_length)
         attention_mask = tf.cast(attention_mask, dtype=tf.float32)
 
         # Prepare head mask if needed
@@ -435,7 +451,7 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer):
         else:
             head_mask = [None] * self.num_hidden_layers
 
-        embedding_output = self.embeddings(input_ids)   # (bs, seq_length, dim)
+        embedding_output = self.embeddings(input_ids, inputs_embeds=inputs_embeds)   # (bs, seq_length, dim)
         tfmr_output = self.transformer([embedding_output, attention_mask, head_mask], training=training)
 
         return tfmr_output # last-layer hidden-state, (all hidden_states), (all attentions)
diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py
index 23866a1a0a..5e416a5e3a 100644
--- a/transformers/modeling_tf_gpt2.py
+++ b/transformers/modeling_tf_gpt2.py
@@ -231,7 +231,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer):
         """
         raise NotImplementedError
 
-    def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+    def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             past = inputs[1] if len(inputs) > 1 else past
@@ -239,7 +239,8 @@ class TFGPT2MainLayer(tf.keras.layers.Layer):
             token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids
             position_ids = inputs[4] if len(inputs) > 4 else position_ids
             head_mask = inputs[5] if len(inputs) > 5 else head_mask
-            assert len(inputs) <= 6, "Too many inputs."
+            inputs_embeds = inputs[6] if len(inputs) > 6 else inputs_embeds
+            assert len(inputs) <= 7, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             past = inputs.get('past', past)
@@ -247,17 +248,28 @@ class TFGPT2MainLayer(tf.keras.layers.Layer):
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 6, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 7, "Too many inputs."
         else:
             input_ids = inputs
 
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = shape_list(input_ids)
+            input_ids = tf.reshape(input_ids, [-1, input_shape[-1]])
+        elif inputs_embeds is not None:
+            input_shape = shape_list(inputs_embeds)[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if past is None:
             past_length = 0
             past = [None] * len(self.h)
         else:
             past_length = shape_list(past[0][0])[-2]
         if position_ids is None:
-            position_ids = tf.range(past_length, shape_list(input_ids)[-1] + past_length, dtype=tf.int32)[tf.newaxis, :]
+            position_ids = tf.range(past_length, input_shape[-1] + past_length, dtype=tf.int32)[tf.newaxis, :]
 
         if attention_mask is not None:
             # We create a 3D attention mask from a 2D tensor mask.
@@ -289,11 +301,10 @@ class TFGPT2MainLayer(tf.keras.layers.Layer):
             head_mask = [None] * self.num_hidden_layers
             # head_mask = tf.constant([0] * self.num_hidden_layers)
 
-        input_shape = shape_list(input_ids)
-        input_ids = tf.reshape(input_ids, [-1, input_shape[-1]])
         position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]])
 
-        inputs_embeds = self.wte(input_ids, mode='embedding')
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids, mode='embedding')
         position_embeds = self.wpe(position_ids)
         if token_type_ids is not None:
             token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]])
@@ -569,7 +580,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
     def get_output_embeddings(self):
         return self.transformer.wte
 
-    def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False):
+    def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, mc_token_ids=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             past = inputs[1] if len(inputs) > 1 else past
@@ -577,8 +588,9 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
             token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids
             position_ids = inputs[4] if len(inputs) > 4 else position_ids
             head_mask = inputs[5] if len(inputs) > 5 else head_mask
-            mc_token_ids = inputs[6] if len(inputs) > 6 else mc_token_ids
-            assert len(inputs) <= 7, "Too many inputs."
+            inputs_embeds = inputs[6] if len(inputs) > 6 else inputs_embeds
+            mc_token_ids = inputs[7] if len(inputs) > 7 else mc_token_ids
+            assert len(inputs) <= 8, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             past = inputs.get('past', past)
@@ -586,21 +598,25 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
             mc_token_ids = inputs.get('mc_token_ids', mc_token_ids)
-            assert len(inputs) <= 7, "Too many inputs."
+            assert len(inputs) <= 8, "Too many inputs."
         else:
             input_ids = inputs
 
-        input_shapes = shape_list(input_ids)
+        if input_ids is not None:
+            input_shapes = shape_list(input_ids)
+        else:
+            input_shapes = shape_list(inputs_embeds)[:-1]
 
         seq_length = input_shapes[-1]
 
-        flat_input_ids = tf.reshape(input_ids, (-1, seq_length))
+        flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None
         flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None
         flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None
         flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None
 
-        flat_inputs = [flat_input_ids, past, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask]
+        flat_inputs = [flat_input_ids, past, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask, inputs_embeds]
 
         transformer_outputs = self.transformer(flat_inputs, training=training)
         hidden_states = transformer_outputs[0]
diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py
index bddd9338b1..c553d92317 100644
--- a/transformers/modeling_tf_openai.py
+++ b/transformers/modeling_tf_openai.py
@@ -229,26 +229,38 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer):
         """
         raise NotImplementedError
 
-    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
             token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
             position_ids = inputs[3] if len(inputs) > 3 else position_ids
             head_mask = inputs[4] if len(inputs) > 4 else head_mask
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds
+            assert len(inputs) <= 6, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 6, "Too many inputs."
         else:
             input_ids = inputs
 
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = shape_list(input_ids)
+            input_ids = tf.reshape(input_ids, [-1, input_shape[-1]])
+        elif inputs_embeds is not None:
+            input_shape = shape_list(inputs_embeds)[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if position_ids is None:
-            position_ids = tf.range(shape_list(input_ids)[-1], dtype=tf.int32)[tf.newaxis, :]
+            position_ids = tf.range(input_shape[-1], dtype=tf.int32)[tf.newaxis, :]
 
         if attention_mask is not None:
             # We create a 3D attention mask from a 2D tensor mask.
@@ -280,11 +292,10 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer):
             head_mask = [None] * self.num_hidden_layers
             # head_mask = tf.constant([0] * self.num_hidden_layers)
 
-        input_shape = shape_list(input_ids)
-        input_ids = tf.reshape(input_ids, [-1, input_shape[-1]])
         position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]])
 
-        inputs_embeds = self.tokens_embed(input_ids, mode='embedding')
+        if inputs_embeds is None:
+            inputs_embeds = self.tokens_embed(input_ids, mode='embedding')
         position_embeds = self.positions_embed(position_ids)
         if token_type_ids is not None:
             token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]])
@@ -533,36 +544,41 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel):
     def get_output_embeddings(self):
         return self.transformer.tokens_embed
 
-    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False):
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, mc_token_ids=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
             token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
             position_ids = inputs[3] if len(inputs) > 3 else position_ids
             head_mask = inputs[4] if len(inputs) > 4 else head_mask
-            mc_token_ids = inputs[5] if len(inputs) > 5 else mc_token_ids
-            assert len(inputs) <= 6, "Too many inputs."
+            inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds
+            mc_token_ids = inputs[6] if len(inputs) > 6 else mc_token_ids
+            assert len(inputs) <= 7, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
             mc_token_ids = inputs.get('mc_token_ids', mc_token_ids)
-            assert len(inputs) <= 6, "Too many inputs."
+            assert len(inputs) <= 7, "Too many inputs."
         else:
             input_ids = inputs
 
-        input_shapes = shape_list(input_ids)
+        if input_ids is not None:
+            input_shapes = shape_list(input_ids)
+        else:
+            input_shapes = shape_list(inputs_embeds)[:-1]
 
         seq_length = input_shapes[-1]
 
-        flat_input_ids = tf.reshape(input_ids, (-1, seq_length))
+        flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None
         flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None
         flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None
         flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None
 
-        flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask]
+        flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask, inputs_embeds]
 
         transformer_outputs = self.transformer(flat_inputs, training=training)
         hidden_states = transformer_outputs[0]
diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py
index c335910dc6..450c0c72f2 100644
--- a/transformers/modeling_tf_roberta.py
+++ b/transformers/modeling_tf_roberta.py
@@ -48,13 +48,17 @@ class TFRobertaEmbeddings(TFBertEmbeddings):
 
     def _embedding(self, inputs, training=False):
         """Applies embedding based on inputs tensor."""
-        input_ids, position_ids, token_type_ids = inputs
+        input_ids, position_ids, token_type_ids, inputs_embeds = inputs
+
+        if input_ids is not None:
+            seq_length = tf.shape(input_ids)[1]
+        else:
+            seq_length = tf.shape(inputs_embeds)[1]
 
-        seq_length = tf.shape(input_ids)[1]
         if position_ids is None:
             position_ids = tf.range(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=tf.int32)[tf.newaxis, :]
 
-        return super(TFRobertaEmbeddings, self)._embedding([input_ids, position_ids, token_type_ids], training=training)
+        return super(TFRobertaEmbeddings, self)._embedding([input_ids, position_ids, token_type_ids, inputs_embeds], training=training)
 
 
 class TFRobertaMainLayer(TFBertMainLayer):
diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py
index 8c2a35b352..8a8d11cfbc 100644
--- a/transformers/modeling_tf_transfo_xl.py
+++ b/transformers/modeling_tf_transfo_xl.py
@@ -430,11 +430,11 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer):
     def _prune_heads(self, heads):
         raise NotImplementedError
 
-    def init_mems(self, data):
+    def init_mems(self, bsz):
         if self.mem_len > 0:
             mems = []
             for i in range(self.n_layer):
-                empty = tf.zeros([self.mem_len, shape_list(data)[1], self.d_model])
+                empty = tf.zeros([self.mem_len, bsz, self.d_model])
                 mems.append(empty)
 
             return mems
@@ -464,28 +464,37 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer):
 
         return new_mems
 
-    def call(self, inputs, mems=None, head_mask=None, training=False):
+    def call(self, inputs, mems=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             mems = inputs[1] if len(inputs) > 1 else mems
             head_mask = inputs[2] if len(inputs) > 2 else head_mask
-            assert len(inputs) <= 3, "Too many inputs."
+            inputs_embeds = inputs[3] if len(inputs) > 3 else inputs_embeds
+            assert len(inputs) <= 4, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             mems = inputs.get('mems', mems)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 3, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 4, "Too many inputs."
         else:
             input_ids = inputs
 
         # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library
         # so we transpose here from shape [bsz, len] to shape [len, bsz]
-        input_ids = tf.transpose(input_ids, perm=(1, 0))
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_ids = tf.transpose(input_ids, perm=(1, 0))
+            qlen, bsz = shape_list(input_ids)
+        elif inputs_embeds is not None:
+            inputs_embeds = tf.transpose(inputs_embeds, perm=(1, 0, 2))
+            qlen, bsz = shape_list(inputs_embeds)[:2]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
 
         if mems is None:
-            mems = self.init_mems(input_ids)
-
-        qlen, bsz = shape_list(input_ids)
+            mems = self.init_mems(bsz)
 
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
@@ -497,7 +506,10 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer):
         else:
             head_mask = [None] * self.n_layer
 
-        word_emb = self.word_emb(input_ids)
+        if inputs_embeds is not None:
+            word_emb = inputs_embeds
+        else:
+            word_emb = self.word_emb(input_ids)
 
         mlen = shape_list(mems[0])[0] if mems is not None else 0
         klen = mlen + qlen
@@ -723,28 +735,33 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel):
     def reset_length(self, tgt_len, ext_len, mem_len):
         self.transformer.reset_length(tgt_len, ext_len, mem_len)
 
-    def init_mems(self, data):
-        return self.transformer.init_mems(data)
+    def init_mems(self, bsz):
+        return self.transformer.init_mems(bsz)
 
-    def call(self, inputs, mems=None, head_mask=None, labels=None, training=False):
+    def call(self, inputs, mems=None, head_mask=None, inputs_embeds=None, labels=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             mems = inputs[1] if len(inputs) > 1 else mems
             head_mask = inputs[2] if len(inputs) > 2 else head_mask
-            labels = inputs[3] if len(inputs) > 3 else labels
-            assert len(inputs) <= 4, "Too many inputs."
+            inputs_embeds = inputs[3] if len(inputs) > 3 else inputs_embeds
+            labels = inputs[4] if len(inputs) > 4 else labels
+            assert len(inputs) <= 5, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             mems = inputs.get('mems', mems)
             head_mask = inputs.get('head_mask', head_mask)
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
             labels = inputs.get('labels', labels)
-            assert len(inputs) <= 4, "Too many inputs."
+            assert len(inputs) <= 5, "Too many inputs."
         else:
             input_ids = inputs
 
-        bsz, tgt_len = shape_list(input_ids)[:2]
+        if input_ids is not None:
+            bsz, tgt_len = shape_list(input_ids)[:2]
+        else:
+            bsz, tgt_len = shape_list(inputs_embeds)[:2]
 
-        transformer_outputs = self.transformer([input_ids, mems, head_mask], training=training)
+        transformer_outputs = self.transformer([input_ids, mems, head_mask, inputs_embeds], training=training)
 
         last_hidden = transformer_outputs[0]
         pred_hid = last_hidden[:, -tgt_len:]
diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py
index 20fbdca732..6f11b0537d 100644
--- a/transformers/modeling_tf_xlm.py
+++ b/transformers/modeling_tf_xlm.py
@@ -291,7 +291,7 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
         raise NotImplementedError
 
     def call(self, inputs, attention_mask=None, langs=None, token_type_ids=None,
-             position_ids=None, lengths=None, cache=None, head_mask=None,
+             position_ids=None, lengths=None, cache=None, head_mask=None, inputs_embeds=None,
              training=False):  # removed: src_enc=None, src_len=None
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
@@ -302,7 +302,8 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
             lengths = inputs[5] if len(inputs) > 5 else lengths
             cache = inputs[6] if len(inputs) > 6 else cache
             head_mask = inputs[7] if len(inputs) > 7 else head_mask
-            assert len(inputs) <= 8, "Too many inputs."
+            inputs_embeds = inputs[8] if len(inputs) > 8 else inputs_embeds
+            assert len(inputs) <= 9, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
@@ -312,16 +313,28 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
             lengths = inputs.get('lengths', lengths)
             cache = inputs.get('cache', cache)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 8, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 9, "Too many inputs."
         else:
             input_ids = inputs
 
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            bs, slen = shape_list(input_ids)
+        elif inputs_embeds is not None:
+            bs, slen = shape_list(inputs_embeds)[:2]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if lengths is None:
-            lengths = tf.reduce_sum(tf.cast(tf.not_equal(input_ids, self.pad_index), dtype=tf.int32), axis=1)
+            if input_ids is not None:
+                lengths = tf.reduce_sum(tf.cast(tf.not_equal(input_ids, self.pad_index), dtype=tf.int32), axis=1)
+            else:
+                lengths = tf.convert_to_tensor([slen]*bs, tf.int32)
         # mask = input_ids != self.pad_index
 
         # check inputs
-        bs, slen = shape_list(input_ids)
         # assert shape_list(lengths)[0] == bs
         tf.debugging.assert_equal(shape_list(lengths)[0], bs)
         # assert lengths.max().item() <= slen
@@ -361,7 +374,7 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
             head_mask = [None] * self.n_layers
 
         # do not recompute cached elements
-        if cache is not None:
+        if cache is not None and input_ids is not None:
             _slen = slen - cache['slen']
             input_ids = input_ids[:, -_slen:]
             position_ids = position_ids[:, -_slen:]
@@ -371,8 +384,10 @@ class TFXLMMainLayer(tf.keras.layers.Layer):
             attn_mask = attn_mask[:, -_slen:]
 
         # embeddings
-        tensor = self.embeddings(input_ids)
-        tensor = tensor + self.position_embeddings(position_ids)
+        if inputs_embeds is None:
+            inputs_embeds = self.embeddings(input_ids)
+
+        tensor = inputs_embeds + self.position_embeddings(position_ids)
         if langs is not None and self.use_lang_emb:
             tensor = tensor + self.lang_embeddings(langs)
         if token_type_ids is not None:
diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py
index 7ab95e7c9f..4733ea8589 100644
--- a/transformers/modeling_tf_xlnet.py
+++ b/transformers/modeling_tf_xlnet.py
@@ -487,7 +487,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer):
         return pos_emb
 
     def call(self, inputs, attention_mask=None, mems=None, perm_mask=None, target_mapping=None,
-            token_type_ids=None, input_mask=None, head_mask=None, training=False):
+            token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
@@ -497,7 +497,8 @@ class TFXLNetMainLayer(tf.keras.layers.Layer):
             token_type_ids = inputs[5] if len(inputs) > 5 else token_type_ids
             input_mask = inputs[6] if len(inputs) > 6 else input_mask
             head_mask = inputs[7] if len(inputs) > 7 else head_mask
-            assert len(inputs) <= 8, "Too many inputs."
+            inputs_embeds = inputs[8] if len(inputs) > 8 else inputs_embeds
+            assert len(inputs) <= 9, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
@@ -507,7 +508,8 @@ class TFXLNetMainLayer(tf.keras.layers.Layer):
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             input_mask = inputs.get('input_mask', input_mask)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 8, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 9, "Too many inputs."
         else:
             input_ids = inputs
 
@@ -515,14 +517,23 @@ class TFXLNetMainLayer(tf.keras.layers.Layer):
         # but we want a unified interface in the library with the batch size on the first dimension
         # so we move here the first dimension (batch) to the end
 
-        input_ids = tf.transpose(input_ids, perm=(1, 0))
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_ids = tf.transpose(input_ids, perm=(1, 0))
+            qlen, bsz = shape_list(input_ids)[:2]
+        elif inputs_embeds is not None:
+            inputs_embeds = tf.transpose(inputs_embeds, perm=(1, 0, 2))
+            qlen, bsz = shape_list(inputs_embeds)[:2]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         token_type_ids = tf.transpose(token_type_ids, perm=(1, 0)) if token_type_ids is not None else None
         input_mask = tf.transpose(input_mask, perm=(1, 0)) if input_mask is not None else None
         attention_mask = tf.transpose(attention_mask, perm=(1, 0)) if attention_mask is not None else None
         perm_mask = tf.transpose(perm_mask, perm=(1, 2, 0)) if perm_mask is not None else None
         target_mapping = tf.transpose(target_mapping, perm=(1, 2, 0)) if target_mapping is not None else None
 
-        qlen, bsz = shape_list(input_ids)[:2]
         mlen = shape_list(mems[0])[0] if mems is not None and mems[0] is not None else 0
         klen = mlen + qlen
 
@@ -573,7 +584,10 @@ class TFXLNetMainLayer(tf.keras.layers.Layer):
             non_tgt_mask = None
 
         ##### Word embeddings and prepare h & g hidden states
-        word_emb_k = self.word_embedding(input_ids)
+        if inputs_embeds is not None:
+            word_emb_k = inputs_embeds
+        else:
+            word_emb_k = self.word_embedding(input_ids)
         output_h = self.dropout(word_emb_k, training=training)
         if target_mapping is not None:
             word_emb_q = tf.tile(self.mask_emb, [tf.shape(target_mapping)[0], bsz, 1])
diff --git a/transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py
index a1715d2568..bcee97435e 100644
--- a/transformers/tests/modeling_tf_bert_test.py
+++ b/transformers/tests/modeling_tf_bert_test.py
@@ -131,10 +131,6 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester):
 
         def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
             model = TFBertModel(config=config)
-            # inputs = {'input_ids': input_ids,
-            #           'attention_mask': input_mask,
-            #           'token_type_ids': token_type_ids}
-            # sequence_output, pooled_output = model(**inputs)
             inputs = {'input_ids': input_ids,
                       'attention_mask': input_mask,
                       'token_type_ids': token_type_ids}
diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py
index ca7beb0925..2bb7cc9c5f 100644
--- a/transformers/tests/modeling_tf_common_test.py
+++ b/transformers/tests/modeling_tf_common_test.py
@@ -411,6 +411,27 @@ class TFCommonTestCases:
                 first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0]
                 self.assertTrue(tf.math.equal(first, second).numpy().all())
 
+        def test_inputs_embeds(self):
+            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+            input_ids = inputs_dict["input_ids"]
+            del inputs_dict["input_ids"]
+
+            for model_class in self.all_model_classes:
+                model = model_class(config)
+
+                wte = model.get_input_embeddings()
+                try:
+                    x = wte(input_ids, mode="embedding")
+                except:
+                    try:
+                        x = wte([input_ids], mode="embedding")
+                    except:
+                        x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
+                # ^^ In our TF models, the input_embeddings can take slightly different forms,
+                # so we try two of them and fall back to just synthetically creating a dummy tensor of ones.
+                inputs_dict["inputs_embeds"] = x
+                outputs = model(inputs_dict)
+
 
 def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None):
     """Creates a random int32 tensor of the shape within the vocab size."""

From 74d0bcb6ff692dbaa52da1fdc2b80ece06f5fbe5 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 12 Nov 2019 15:27:57 -0500
Subject: [PATCH 159/269] Fix special tokens addition in decoder

---
 .../tests/tokenization_tests_commons.py       | 20 +++++++++++++++++++
 transformers/tokenization_utils.py            |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py
index a921696b77..fdaf8cc137 100644
--- a/transformers/tests/tokenization_tests_commons.py
+++ b/transformers/tests/tokenization_tests_commons.py
@@ -160,6 +160,26 @@ class CommonTestCases:
             self.assertEqual(tokens[0], tokenizer.eos_token_id)
             self.assertEqual(tokens[-2], tokenizer.pad_token_id)
 
+        def test_add_special_tokens(self):
+            tokenizer = self.get_tokenizer()
+            input_text, output_text = self.get_input_output_texts()
+
+            special_token = "[SPECIAL TOKEN]"
+
+            tokenizer.add_special_tokens({"cls_token": special_token})
+            encoded_special_token = tokenizer.encode(special_token, add_special_tokens=False)
+            assert len(encoded_special_token) == 1
+
+            text = " ".join([input_text, special_token, output_text])
+            encoded = tokenizer.encode(text, add_special_tokens=False)
+
+            input_encoded = tokenizer.encode(input_text, add_special_tokens=False)
+            output_encoded = tokenizer.encode(output_text, add_special_tokens=False)
+            special_token_id = tokenizer.encode(special_token, add_special_tokens=False)
+            assert encoded == input_encoded + special_token_id + output_encoded
+
+            decoded = tokenizer.decode(encoded, skip_special_tokens=True)
+            assert special_token not in decoded
 
         def test_required_methods_tokenizer(self):
             tokenizer = self.get_tokenizer()
diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py
index cd14cc4582..f37f6f3206 100644
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -1055,7 +1055,7 @@ class PreTrainedTokenizer(object):
             class attributes (cls_token, unk_token...).
         """
         all_toks = self.all_special_tokens
-        all_ids = list(self._convert_token_to_id(t) for t in all_toks)
+        all_ids = self.convert_tokens_to_ids(all_toks)
         return all_ids
 
     @staticmethod

From 7627dde1f8888faf5b05a8f1dbbc2271096cc1e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=B0brahim=20Ethem=20Demirci?= <dmrc_wr@hotmail.com>
Date: Thu, 14 Nov 2019 17:06:15 +0300
Subject: [PATCH 160/269] sum() is the leanest method to flatten a string list,
 so it's been replaced by itertools.chain.from_iterable()

---
 transformers/tokenization_utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py
index cd14cc4582..1eebae08a7 100644
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -21,6 +21,7 @@ import os
 import json
 import six
 import copy
+import itertools
 from io import open
 
 from .file_utils import cached_path, is_tf_available, is_torch_available
@@ -641,9 +642,9 @@ class PreTrainedTokenizer(object):
                         tokenized_text += [sub_text]
                 text_list = tokenized_text
 
-            return sum((self._tokenize(token, **kwargs) if token not \
+            return list(itertools.chain.from_iterable((self._tokenize(token, **kwargs) if token not \
                     in self.added_tokens_encoder and token not in self.all_special_tokens \
-                    else [token] for token in tokenized_text), [])
+                    else [token] for token in tokenized_text)))
 
         added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens
         tokenized_text = split_on_tokens(added_tokens, text)

From 022525b0031bcdbbb62d1223f75919983f2ac426 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Tue, 12 Nov 2019 11:08:47 +0100
Subject: [PATCH 161/269] replace LambdaLR scheduler wrappers by function

Custom schedulers are currently initiated by wrapping Pytorch's LambdaLR
class and passing a method of the wrapping class to the __init__
function of LambdaLR. This approach is not appropriate for several
reasons:

1. one does not need to define a class when it only defines a
__init__() method;
2. instantiating the parent class by passing a method of the child class
creates a cyclical reference which leads to memory leaks. See issues #1742 and #1134.

In this commit we replace the wrapper classes with functions that
instantiate `LambdaLR` with a custom learning rate function. We use a
closure to specify the parameter of the latter. We also do a bit of
renaming within the function to explicit the behaviour and removed
docstrings that were subsequently not necessary.
---
 transformers/__init__.py                |   4 +-
 transformers/optimization.py            | 108 +++++++++---------------
 transformers/tests/optimization_test.py |  29 ++++---
 3 files changed, 61 insertions(+), 80 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 53f3c39dc7..426f3bd3a2 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -97,8 +97,8 @@ if is_torch_available():
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
     # Optimization
-    from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule,
-                               WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule)
+    from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup,
+                               get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup)
 
 
 # TensorFlow
diff --git a/transformers/optimization.py b/transformers/optimization.py
index a48b5fea54..99e6cc75e4 100644
--- a/transformers/optimization.py
+++ b/transformers/optimization.py
@@ -23,89 +23,65 @@ from torch.optim.lr_scheduler import LambdaLR
 
 logger = logging.getLogger(__name__)
 
-class ConstantLRSchedule(LambdaLR):
-    """ Constant learning rate schedule.
+
+def get_constant_schedule(optimizer, last_epoch=-1):
+    """ Create a schedule with a constant learning rate.
     """
-    def __init__(self, optimizer, last_epoch=-1):
-        super(ConstantLRSchedule, self).__init__(optimizer, lambda _: 1.0, last_epoch=last_epoch)
+    return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch)
 
 
-class WarmupConstantSchedule(LambdaLR):
-    """ Linear warmup and then constant.
-        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
-        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
-        Keeps multiplicative variable equal to 1. after warmup_steps.
+def get_constant_schedule_with_warmup(optimizer, num_warmup_steps, last_epoch=-1):
+    """ Create a schedule with a constant learning rate preceded by a warmup
+    period during which the learning rate increases linearly between 0 and 1.
     """
-    def __init__(self, optimizer, warmup_steps, last_epoch=-1):
-        self.warmup_steps = warmup_steps
-        super(WarmupConstantSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)
-
-    def lr_lambda(self, step):
-        if step < self.warmup_steps:
-            return float(step) / float(max(1.0, self.warmup_steps))
+    def lr_lambda(current_step):
+        if current_step < num_warmup_steps:
+            return float(current_step) / float(max(1.0, num_warmup_steps))
         return 1.
 
+    return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch)
 
-class WarmupLinearSchedule(LambdaLR):
-    """ Linear warmup and then linear decay.
-        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
-        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
-        Linearly decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps.
+
+def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):
+    """ Create a schedule with a learning rate that decreases linearly after
+    linearly increasing during a warmup period.
     """
-    def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1):
-        self.warmup_steps = warmup_steps
-        self.t_total = t_total
-        super(WarmupLinearSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)
+    def lr_lambda(current_step):
+        if current_step < num_warmup_steps:
+            return float(current_step) / float(max(1, num_warmup_steps))
+        return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)))
 
-    def lr_lambda(self, step):
-        if step < self.warmup_steps:
-            return float(step) / float(max(1, self.warmup_steps))
-        return max(0.0, float(self.t_total - step) / float(max(1.0, self.t_total - self.warmup_steps)))
+    return LambdaLR(optimizer, lr_lambda, last_epoch)
 
 
-class WarmupCosineSchedule(LambdaLR):
-    """ Linear warmup and then cosine decay.
-        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
-        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
-        Decreases the multiplicative variable from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.
-        If `cycles` (default=0.5) is different from default, then the multiplicative variable follows cosine function after warmup.
+def get_cosine_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=.5, last_epoch=-1):
+    """ Create a schedule with a learning rate that decreases following the
+    values of the cosine function between 0 and `pi * cycles` after a warmup
+    period during which it increases linearly between 0 and 1.
     """
-    def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):
-        self.warmup_steps = warmup_steps
-        self.t_total = t_total
-        self.cycles = cycles
-        super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)
+    def lr_lambda(current_step):
+        if current_step < num_warmup_steps:
+            return float(current_step) / float(max(1, num_warmup_steps))
+        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))
+        return max(0., 0.5 * (1. + math.cos(math.pi * float(num_cycles) * 2. * progress)))
 
-    def lr_lambda(self, step):
-        if step < self.warmup_steps:
-            return float(step) / float(max(1.0, self.warmup_steps))
-        # progress after warmup
-        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))
-        return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))
+    return LambdaLR(optimizer, lr_lambda, last_epoch)
 
 
-class WarmupCosineWithHardRestartsSchedule(LambdaLR):
-    """ Linear warmup and then cosine cycles with hard restarts.
-        Multiplies the learning rate defined in the optimizer by a dynamic variable determined by the current step.
-        Linearly increases the multiplicative variable from 0. to 1. over `warmup_steps` training steps.
-        If `cycles` (default=1.) is different from default, learning rate  follows `cycles` times a cosine decaying
-        learning rate (with hard restarts).
+def get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, num_cycles=1., last_epoch=-1):
+    """ Create a schedule with a learning rate that decreases following the
+    values of the cosine function with several hard restarts, after a warmup
+    period during which it increases linearly between 0 and 1.
     """
-    def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1):
-        self.warmup_steps = warmup_steps
-        self.t_total = t_total
-        self.cycles = cycles
-        super(WarmupCosineWithHardRestartsSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)
-
-    def lr_lambda(self, step):
-        if step < self.warmup_steps:
-            return float(step) / float(max(1, self.warmup_steps))
-        # progress after warmup
-        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))
-        if progress >= 1.0:
-            return 0.0
-        return max(0.0, 0.5 * (1. + math.cos(math.pi * ((float(self.cycles) * progress) % 1.0))))
+    def lr_lambda(current_step):
+        if current_step < num_warmup_steps:
+            return float(current_step) / float(max(1, num_warmup_steps))
+        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))
+        if progress >= 1.:
+            return 0.
+        return max(0., 0.5 * (1. + math.cos(math.pi * ((float(num_cycles) * progress) % 1.))))
 
+    return LambdaLR(optimizer, lr_lambda, last_epoch)
 
 
 class AdamW(Optimizer):
diff --git a/transformers/tests/optimization_test.py b/transformers/tests/optimization_test.py
index 84dbaca52a..ab9afbfcf7 100644
--- a/transformers/tests/optimization_test.py
+++ b/transformers/tests/optimization_test.py
@@ -25,8 +25,12 @@ from transformers import is_torch_available
 if is_torch_available():
     import torch
 
-    from transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule,
-                                    WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule)
+    from transformers import (AdamW,
+                              get_constant_schedule,
+                              get_constant_schedule_with_warmup,
+                              get_cosine_schedule_with_warmup,
+                              get_cosine_with_hard_restarts_schedule_with_warmup,
+                              get_linear_schedule_with_warmup)
 else:
     pytestmark = pytest.mark.skip("Require Torch")
 
@@ -87,59 +91,60 @@ class ScheduleInitTest(unittest.TestCase):
             self.assertAlmostEqual(a, b, delta=tol)
 
     def test_constant_scheduler(self):
-        scheduler = ConstantLRSchedule(self.optimizer)
+        scheduler = get_constant_schedule(self.optimizer)
         lrs = unwrap_schedule(scheduler, self.num_steps)
         expected_learning_rates = [10.] * self.num_steps
         self.assertEqual(len(lrs[0]), 1)
         self.assertListEqual([l[0] for l in lrs], expected_learning_rates)
 
-        scheduler = ConstantLRSchedule(self.optimizer)
+        scheduler = get_constant_schedule(self.optimizer)
         lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps)
         self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2])
 
     def test_warmup_constant_scheduler(self):
-        scheduler = WarmupConstantSchedule(self.optimizer, warmup_steps=4)
+        scheduler = get_constant_schedule_with_warmup(self.optimizer, num_warmup_steps=4)
         lrs = unwrap_schedule(scheduler, self.num_steps)
         expected_learning_rates = [2.5, 5.0, 7.5, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]
         self.assertEqual(len(lrs[0]), 1)
         self.assertListEqual([l[0] for l in lrs], expected_learning_rates)
 
-        scheduler = WarmupConstantSchedule(self.optimizer, warmup_steps=4)
+        scheduler = get_constant_schedule_with_warmup(self.optimizer, num_warmup_steps=4)
         lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps)
         self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2])
 
     def test_warmup_linear_scheduler(self):
-        scheduler = WarmupLinearSchedule(self.optimizer, warmup_steps=2, t_total=10)
+        scheduler = get_linear_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10)
         lrs = unwrap_schedule(scheduler, self.num_steps)
         expected_learning_rates = [5.0, 10.0, 8.75, 7.5, 6.25, 5.0, 3.75, 2.5, 1.25, 0.0]
         self.assertEqual(len(lrs[0]), 1)
         self.assertListEqual([l[0] for l in lrs], expected_learning_rates)
 
-        scheduler = WarmupLinearSchedule(self.optimizer, warmup_steps=2, t_total=10)
+        scheduler = get_linear_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10)
         lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps)
         self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2])
 
     def test_warmup_cosine_scheduler(self):
-        scheduler = WarmupCosineSchedule(self.optimizer, warmup_steps=2, t_total=10)
+        scheduler = get_cosine_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10)
         lrs = unwrap_schedule(scheduler, self.num_steps)
         expected_learning_rates = [5.0, 10.0, 9.61, 8.53, 6.91, 5.0, 3.08, 1.46, 0.38, 0.0]
         self.assertEqual(len(lrs[0]), 1)
         self.assertListAlmostEqual([l[0] for l in lrs], expected_learning_rates, tol=1e-2)
 
-        scheduler = WarmupCosineSchedule(self.optimizer, warmup_steps=2, t_total=10)
+        scheduler = get_cosine_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_training_steps=10)
         lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps)
         self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2])
 
     def test_warmup_cosine_hard_restart_scheduler(self):
-        scheduler = WarmupCosineWithHardRestartsSchedule(self.optimizer, warmup_steps=2, cycles=2, t_total=10)
+        scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_cycles=2, num_training_steps=10)
         lrs = unwrap_schedule(scheduler, self.num_steps)
         expected_learning_rates = [5.0, 10.0, 8.53, 5.0, 1.46, 10.0, 8.53, 5.0, 1.46, 0.0]
         self.assertEqual(len(lrs[0]), 1)
         self.assertListAlmostEqual([l[0] for l in lrs], expected_learning_rates, tol=1e-2)
 
-        scheduler = WarmupCosineWithHardRestartsSchedule(self.optimizer, warmup_steps=2, cycles=2, t_total=10)
+        scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(self.optimizer, num_warmup_steps=2, num_cycles=2, num_training_steps=10)
         lrs_2 = unwrap_and_save_reload_schedule(scheduler, self.num_steps)
         self.assertListEqual([l[0] for l in lrs], [l[0] for l in lrs_2])
 
+
 if __name__ == "__main__":
     unittest.main()

From e18f786cd57c8cb84a09c6ce5e0d7de0dd8b106e Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 14 Nov 2019 10:06:00 -0500
Subject: [PATCH 162/269] Quickstart example showcasing past

---
 docs/source/quickstart.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md
index ccba75e7c0..530aff8eb0 100644
--- a/docs/source/quickstart.md
+++ b/docs/source/quickstart.md
@@ -188,3 +188,35 @@ assert predicted_text == 'Who was Jim Henson? Jim Henson was a man'
 ```
 
 Examples for each model class of each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [documentation](#documentation).
+
+#### Using the past
+
+GPT-2 as well as some other models (GPT, XLNet, Transfo-XL, CTRL) make use of a `past` or `mems` attribute which can be used to prevent re-computing the key/value pairs when using sequential decoding. It is useful when generating sequences as a big part of the attention mechanism benefits from previous computations.
+
+Here is a fully-working example using the `past` with `GPT2LMHeadModel` and argmax decoding (which should only be used as an example, as argmax decoding introduces a lot of repetition):
+
+```python
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+import torch
+
+tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+model = GPT2LMHeadModel.from_pretrained('gpt2')
+
+generated = tokenizer.encode("The Manhattan bridge")
+context = torch.tensor([generated])
+past = None
+
+for i in range(100):
+    print(i)
+    output, past = model(context, past=past)
+    token = torch.argmax(output[0, :])
+
+    generated += [token.tolist()]
+    context = token.unsqueeze(0)
+
+sequence = tokenizer.decode(generated)
+
+print(sequence)
+```
+
+The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`.
\ No newline at end of file

From a67e7478894c477822760ad7f9933a7d78aa27bd Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 14 Nov 2019 10:30:22 -0500
Subject: [PATCH 163/269] Reorganized max_len warning

---
 transformers/tokenization_utils.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py
index cd14cc4582..c5f469800b 100644
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -671,10 +671,6 @@ class PreTrainedTokenizer(object):
         ids = []
         for token in tokens:
             ids.append(self._convert_token_to_id_with_added_voc(token))
-        if len(ids) > self.max_len:
-            logger.warning("Token indices sequence length is longer than the specified maximum sequence length "
-                           "for this model ({} > {}). Running this sequence through the model will result in "
-                           "indexing errors".format(len(ids), self.max_len))
         return ids
 
     def _convert_token_to_id_with_added_voc(self, token):
@@ -877,6 +873,11 @@ class PreTrainedTokenizer(object):
             encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length]
             encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length]
 
+        if max_length is None and len(encoded_inputs["input_ids"]) > self.max_len:
+            logger.warning("Token indices sequence length is longer than the specified maximum sequence length "
+                           "for this model ({} > {}). Running this sequence through the model will result in "
+                           "indexing errors".format(len(ids), self.max_len))
+                           
         return encoded_inputs
 
     def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0):

From d7929899daaca2d0b910dcef1181ea496f0b4909 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 14 Nov 2019 10:49:00 -0500
Subject: [PATCH 164/269] Specify checkpoint in saved file for
 run_lm_finetuning.py

---
 examples/run_lm_finetuning.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 2044cfe9e8..a143d55894 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -63,10 +63,10 @@ MODEL_CLASSES = {
 
 
 class TextDataset(Dataset):
-    def __init__(self, tokenizer, file_path='train', block_size=512):
+    def __init__(self, tokenizer, args, file_path='train', block_size=512):
         assert os.path.isfile(file_path)
         directory, filename = os.path.split(file_path)
-        cached_features_file = os.path.join(directory, 'cached_lm_' + str(block_size) + '_' + filename)
+        cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename)
 
         if os.path.exists(cached_features_file):
             logger.info("Loading features from cached file %s", cached_features_file)
@@ -99,7 +99,7 @@ class TextDataset(Dataset):
 
 
 def load_and_cache_examples(args, tokenizer, evaluate=False):
-    dataset = TextDataset(tokenizer, file_path=args.eval_data_file if evaluate else args.train_data_file, block_size=args.block_size)
+    dataset = TextDataset(tokenizer, args, file_path=args.eval_data_file if evaluate else args.train_data_file, block_size=args.block_size)
     return dataset
 
 

From 2276bf69b771763a2553eccd5d70c2b7331b1f99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Thu, 14 Nov 2019 18:00:14 +0100
Subject: [PATCH 165/269] update the examples, docs and template

---
 README.md                                         |  8 ++++----
 docs/source/main_classes/optimizer_schedules.rst  | 14 +++++---------
 docs/source/migration.md                          |  8 ++++----
 examples/contrib/run_openai_gpt.py                |  4 ++--
 examples/contrib/run_swag.py                      |  4 ++--
 examples/distillation/distiller.py                |  8 ++++----
 examples/distillation/run_squad_w_distillation.py |  4 ++--
 examples/run_glue.py                              |  4 ++--
 examples/run_lm_finetuning.py                     |  4 ++--
 examples/run_multiple_choice.py                   |  4 ++--
 examples/run_ner.py                               |  4 ++--
 examples/run_squad.py                             |  4 ++--
 templates/adding_a_new_example_script/run_xxx.py  |  4 ++--
 13 files changed, 35 insertions(+), 39 deletions(-)

diff --git a/README.md b/README.md
index 40b08583b1..a22a3c83f5 100644
--- a/README.md
+++ b/README.md
@@ -521,12 +521,12 @@ Here is a conversion examples from `BertAdam` with a linear warmup and decay sch
 # Parameters:
 lr = 1e-3
 max_grad_norm = 1.0
-num_total_steps = 1000
+num_training_steps = 1000
 num_warmup_steps = 100
-warmup_proportion = float(num_warmup_steps) / float(num_total_steps)  # 0.1
+warmup_proportion = float(num_warmup_steps) / float(num_training_steps)  # 0.1
 
 ### Previously BertAdam optimizer was instantiated like this:
-optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps)
+optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_training_steps)
 ### and used like this:
 for batch in train_data:
     loss = model(batch)
@@ -535,7 +535,7 @@ for batch in train_data:
 
 ### In Transformers, optimizer and schedules are splitted and instantiated like this:
 optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False
-scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler
+scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  # PyTorch scheduler
 ### and used like this:
 for batch in train_data:
     model.train()
diff --git a/docs/source/main_classes/optimizer_schedules.rst b/docs/source/main_classes/optimizer_schedules.rst
index ff0c9e6929..b30a2e0e2e 100644
--- a/docs/source/main_classes/optimizer_schedules.rst
+++ b/docs/source/main_classes/optimizer_schedules.rst
@@ -18,19 +18,17 @@ Schedules
 Learning Rate Schedules
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: transformers.ConstantLRSchedule
-    :members:
+.. autofunction:: transformers.get_constant_schedule
 
 
-.. autoclass:: transformers.WarmupConstantSchedule
-    :members:
+.. autofunction:: transformers.get_constant_schedule_with_warmup
 
 .. image:: /imgs/warmup_constant_schedule.png
     :target: /imgs/warmup_constant_schedule.png
     :alt:
 
 
-.. autoclass:: transformers.WarmupCosineSchedule
+.. autofunction:: transformers.get_cosine_schedule_with_warmup
     :members:
 
 .. image:: /imgs/warmup_cosine_schedule.png
@@ -38,8 +36,7 @@ Learning Rate Schedules
     :alt:
 
 
-.. autoclass:: transformers.WarmupCosineWithHardRestartsSchedule
-    :members:
+.. autofunction:: transformers.get_cosine_with_hard_restarts_schedule_with_warmup
 
 .. image:: /imgs/warmup_cosine_hard_restarts_schedule.png
     :target: /imgs/warmup_cosine_hard_restarts_schedule.png
@@ -47,8 +44,7 @@ Learning Rate Schedules
 
 
 
-.. autoclass:: transformers.WarmupLinearSchedule
-    :members:
+.. autofunction:: transformers.get_linear_schedule_with_warmup
 
 .. image:: /imgs/warmup_linear_schedule.png
     :target: /imgs/warmup_linear_schedule.png
diff --git a/docs/source/migration.md b/docs/source/migration.md
index 553a79c82b..d04b66d5e4 100644
--- a/docs/source/migration.md
+++ b/docs/source/migration.md
@@ -84,12 +84,12 @@ Here is a conversion examples from `BertAdam` with a linear warmup and decay sch
 # Parameters:
 lr = 1e-3
 max_grad_norm = 1.0
-num_total_steps = 1000
+num_training_steps = 1000
 num_warmup_steps = 100
-warmup_proportion = float(num_warmup_steps) / float(num_total_steps)  # 0.1
+warmup_proportion = float(num_warmup_steps) / float(num_training_steps)  # 0.1
 
 ### Previously BertAdam optimizer was instantiated like this:
-optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps)
+optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, num_training_steps=num_training_steps)
 ### and used like this:
 for batch in train_data:
     loss = model(batch)
@@ -98,7 +98,7 @@ for batch in train_data:
 
 ### In Transformers, optimizer and schedules are splitted and instantiated like this:
 optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False
-scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler
+scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  # PyTorch scheduler
 ### and used like this:
 for batch in train_data:
     loss = model(batch)
diff --git a/examples/contrib/run_openai_gpt.py b/examples/contrib/run_openai_gpt.py
index 7eb1b0be76..2d165a91e3 100644
--- a/examples/contrib/run_openai_gpt.py
+++ b/examples/contrib/run_openai_gpt.py
@@ -41,7 +41,7 @@ from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
 
 from transformers import (OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer,
                                      AdamW, cached_path, WEIGHTS_NAME, CONFIG_NAME,
-                                     WarmupLinearSchedule)
+                                     get_linear_schedule_with_warmup)
 
 ROCSTORIES_URL = "https://s3.amazonaws.com/datasets.huggingface.co/ROCStories.tar.gz"
 
@@ -211,7 +211,7 @@ def main():
             {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
             ]
         optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
 
     if args.do_train:
         nb_tr_steps, tr_loss, exp_average_loss = 0, 0, None
diff --git a/examples/contrib/run_swag.py b/examples/contrib/run_swag.py
index 8494c5fad9..5de93db7fe 100644
--- a/examples/contrib/run_swag.py
+++ b/examples/contrib/run_swag.py
@@ -42,7 +42,7 @@ from tqdm import tqdm, trange
 from transformers import (WEIGHTS_NAME, BertConfig,
                                   BertForMultipleChoice, BertTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 logger = logging.getLogger(__name__)
 
@@ -322,7 +322,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py
index d51bdae77f..0442072e84 100644
--- a/examples/distillation/distiller.py
+++ b/examples/distillation/distiller.py
@@ -35,7 +35,7 @@ try:
 except:
     from tensorboardX import SummaryWriter
 
-from transformers import WarmupLinearSchedule
+from transformers import get_linear_schedule_with_warmup
 
 from utils import logger
 from lm_seqs_dataset import LmSeqsDataset
@@ -137,9 +137,9 @@ class Distiller:
                                betas=(0.9, 0.98))
 
         warmup_steps = math.ceil(num_train_optimization_steps * params.warmup_prop)
-        self.scheduler = WarmupLinearSchedule(self.optimizer,
-                                                warmup_steps=warmup_steps,
-                                                t_total=num_train_optimization_steps)
+        self.scheduler = get_linear_schedule_with_warmup(self.optimizer,
+                                                num_warmup_steps=warmup_steps,
+                                                num_training_steps=num_train_optimization_steps)
 
         if self.fp16:
             try:
diff --git a/examples/distillation/run_squad_w_distillation.py b/examples/distillation/run_squad_w_distillation.py
index 7c662df010..70b65dc1b8 100644
--- a/examples/distillation/run_squad_w_distillation.py
+++ b/examples/distillation/run_squad_w_distillation.py
@@ -46,7 +46,7 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   XLNetTokenizer,
                                   DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 from ..utils_squad import (read_squad_examples, convert_examples_to_features,
                          RawResult, write_predictions,
@@ -101,7 +101,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/examples/run_glue.py b/examples/run_glue.py
index 1558a812c3..27048ad565 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -49,7 +49,7 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   DistilBertForSequenceClassification,
                                   DistilBertTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_output_modes as output_modes
@@ -100,7 +100,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 2044cfe9e8..0085aee727 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -42,7 +42,7 @@ except:
 
 from tqdm import tqdm, trange
 
-from transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule,
+from transformers import (WEIGHTS_NAME, AdamW, get_linear_schedule_with_warmup,
                                   BertConfig, BertForMaskedLM, BertTokenizer,
                                   GPT2Config, GPT2LMHeadModel, GPT2Tokenizer,
                                   OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer,
@@ -185,7 +185,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py
index 638bbe74f1..544014fb66 100644
--- a/examples/run_multiple_choice.py
+++ b/examples/run_multiple_choice.py
@@ -43,7 +43,7 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   XLNetTokenizer, RobertaConfig,
                                   RobertaForMultipleChoice, RobertaTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 from utils_multiple_choice import (convert_examples_to_features, processors)
 
@@ -101,7 +101,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/examples/run_ner.py b/examples/run_ner.py
index b35d8298fe..0077080aec 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -33,7 +33,7 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer
 from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer
 
@@ -80,7 +80,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id):
         {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0}
     ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/examples/run_squad.py b/examples/run_squad.py
index d9dc2abfde..b954a8b8b9 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -45,7 +45,7 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   XLNetTokenizer,
                                   DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 from utils_squad import (read_squad_examples, convert_examples_to_features,
                          RawResult, write_predictions,
@@ -100,7 +100,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
diff --git a/templates/adding_a_new_example_script/run_xxx.py b/templates/adding_a_new_example_script/run_xxx.py
index 489dcb19c7..77ce587a54 100644
--- a/templates/adding_a_new_example_script/run_xxx.py
+++ b/templates/adding_a_new_example_script/run_xxx.py
@@ -43,7 +43,7 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   XLNetTokenizer,
                                   DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 from utils_squad import (read_squad_examples, convert_examples_to_features,
                          RawResult, write_predictions,
@@ -98,7 +98,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp

From 8f8d69716a1a61c5bc2d9cdcdb9147cfeb8441d1 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 14 Nov 2019 14:48:21 -0500
Subject: [PATCH 166/269] [CI][DOC] Don't rebuild if folder exists.

---
 .circleci/deploy.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 2bff0102ae..7eea0d0113 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -5,8 +5,12 @@ function deploy_doc(){
 	git checkout $1
 	if [ ! -z "$2" ] 
 	then
-		echo "Pushing version" $2
-		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+		if [ -d "$DIRECTORY" ]; then
+			echo "Directory" $2 "already exists"
+		else
+			echo "Pushing version" $2
+			make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+		fi
 	else
 		echo "Pushing master"
 		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir

From be7f2aacce6d2e9e6a4a6a9e04b99ad0e660d703 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 14 Nov 2019 14:54:44 -0500
Subject: [PATCH 167/269] [CI][DOC] Don't rebuild if folder exists - Correct
 directory.

---
 .circleci/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 7eea0d0113..98151963d8 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -5,7 +5,7 @@ function deploy_doc(){
 	git checkout $1
 	if [ ! -z "$2" ] 
 	then
-		if [ -d "$DIRECTORY" ]; then
+		if [ -d "$dir/$2" ]; then
 			echo "Directory" $2 "already exists"
 		else
 			echo "Pushing version" $2

From 05db5bc1afea196e548ae3214d3413c321fcfda1 Mon Sep 17 00:00:00 2001
From: Thomas Wolf <thomwolf@users.noreply.github.com>
Date: Thu, 14 Nov 2019 22:40:22 +0100
Subject: [PATCH 168/269] added small comparison between BERT, RoBERTa and
 DistilBERT

---
 examples/README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/examples/README.md b/examples/README.md
index 2b66b92f1a..abb4cb6e5a 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -554,6 +554,16 @@ On the test dataset the following results could be achieved:
 10/04/2019 00:42:42 - INFO - __main__ -     recall = 0.8624150210424085
 ```
 
+### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased)
+
+Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) with the same hyperparameters as specified in the [example documentation](https://huggingface.co/transformers/examples.html#named-entity-recognition) (one run):
+
+| Model | F-Score Dev | F-Score Test
+| --------------------------------- | ------- | --------
+| `bert-large-cased`            | 95.59 | 91.70
+| `roberta-large`                  | 95.96 | 91.87
+| `distilbert-base-uncased` | 94.34 | 90.32
+
 ## Abstractive summarization
 
 Based on the script

From 14b3aa3b3c300eb1fcc4f3a0c046c87bdabe0afd Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Fri, 8 Nov 2019 16:47:21 -0800
Subject: [PATCH 169/269] Add tokenization_camembert.py

---
 transformers/tokenization_camembert.py | 120 +++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 transformers/tokenization_camembert.py

diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
new file mode 100644
index 0000000000..9facf7d911
--- /dev/null
+++ b/transformers/tokenization_camembert.py
@@ -0,0 +1,120 @@
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization classes for Camembert model."""
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import sentencepiece as spm
+from transformers.tokenization_utils import PreTrainedTokenizer
+
+
+class CamembertTokenizer(PreTrainedTokenizer):
+    """
+        Adapted from RobertaTokenizer and XLNetTokenizer
+        SentencePiece based tokenizer. Peculiarities:
+
+            - requires `SentencePiece <https://github.com/google/sentencepiece>`_
+    """
+    vocab_files_names = {'vocab_file': None}
+
+    def __init__(self, vocab_file, bos_token="<s>", eos_token="</s>", sep_token="</s>",
+                 cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>', **kwargs):
+        super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
+                                                 sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
+                                                 mask_token=mask_token, **kwargs)
+        self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
+        self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(str(vocab_file))
+        # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
+        # sentencepiece vocabulary (this is the case for <s> and </s>
+        self.fairseq_tokens_to_ids = {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
+        self.fairseq_offset = len(self.fairseq_tokens_to_ids)
+        self.fairseq_tokens_to_ids['<mask>'] = len(self.sp_model) + len(self.fairseq_tokens_to_ids)
+        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        by concatenating and adding special tokens.
+        A RoBERTa sequence has the following format:
+            single sequence: <s> X </s>
+            pair of sequences: <s> A </s></s> B </s>
+        """
+        if token_ids_1 is None:
+            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+        cls = [self.cls_token_id]
+        sep = [self.sep_token_id]
+        return cls + token_ids_0 + sep + sep + token_ids_1 + sep
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
+
+        Args:
+            token_ids_0: list of ids (must not contain special tokens)
+            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
+                for sequence pairs
+            already_has_special_tokens: (default False) Set to True if the token list is already formated with
+                special tokens for the model
+
+        Returns:
+            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+        """
+        if already_has_special_tokens:
+            if token_ids_1 is not None:
+                raise ValueError("You should not supply a second sequence if the provided sequence of "
+                                 "ids is already formated with special tokens for the model.")
+            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
+
+        if token_ids_1 is None:
+            return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
+        A RoBERTa sequence pair mask has the following format:
+        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence
+
+        if token_ids_1 is None, only returns the first portion of the mask (0's).
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+
+        if token_ids_1 is None:
+            return len(cls + token_ids_0 + sep) * [0]
+        return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1]
+
+    @property
+    def vocab_size(self):
+        return self.fairseq_offset + len(self.sp_model)
+
+    def _tokenize(self, text):
+        return self.sp_model.EncodeAsPieces(text)
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str/unicode) in an id using the vocab. """
+        if token in self.fairseq_tokens_to_ids:
+            return self.fairseq_tokens_to_ids[token]
+        return self.fairseq_offset + self.sp_model.PieceToId(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (string/unicode) using the vocab."""
+        if index in self.fairseq_ids_to_tokens:
+            return self.fairseq_ids_to_tokens[index]
+        return self.sp_model.IdToPiece(index - self.fairseq_offset)

From 6e72fd094c98901cc90d146d3fe3cd5a0e879911 Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Fri, 8 Nov 2019 17:09:48 -0800
Subject: [PATCH 170/269] Add demo_camembert.py

---
 examples/demo_camembert.py | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 examples/demo_camembert.py

diff --git a/examples/demo_camembert.py b/examples/demo_camembert.py
new file mode 100644
index 0000000000..df28f4f267
--- /dev/null
+++ b/examples/demo_camembert.py
@@ -0,0 +1,59 @@
+from pathlib import Path
+import tarfile
+import urllib.request
+
+import torch
+
+from transformers.tokenization_camembert import CamembertTokenizer
+from transformers.modeling_roberta import RobertaForMaskedLM
+
+
+def fill_mask(masked_input, model, tokenizer, topk=5):
+    # Adapted from https://github.com/pytorch/fairseq/blob/master/fairseq/models/roberta/hub_interface.py
+    assert masked_input.count('<mask>') == 1
+    input_ids = torch.tensor(tokenizer.encode(masked_input, add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+    logits = model(input_ids)[0]  # The last hidden-state is the first element of the output tuple
+    masked_index = (input_ids.squeeze() == tokenizer.mask_token_id).nonzero().item()
+    logits = logits[0, masked_index, :]
+    prob = logits.softmax(dim=0)
+    values, indices = prob.topk(k=topk, dim=0)
+    topk_predicted_token_bpe = ' '.join([tokenizer.convert_ids_to_tokens(indices[i].item())
+                                         for i in range(len(indices))])
+    masked_token = tokenizer.mask_token
+    topk_filled_outputs = []
+    for index, predicted_token_bpe in enumerate(topk_predicted_token_bpe.split(' ')):
+        predicted_token = predicted_token_bpe.replace('\u2581', ' ')
+        if " {0}".format(masked_token) in masked_input:
+            topk_filled_outputs.append((
+                masked_input.replace(
+                    ' {0}'.format(masked_token), predicted_token
+                ),
+                values[index].item(),
+                predicted_token,
+            ))
+        else:
+            topk_filled_outputs.append((
+                masked_input.replace(masked_token, predicted_token),
+                values[index].item(),
+                predicted_token,
+            ))
+    return topk_filled_outputs
+
+
+model_path = Path('camembert.v0.pytorch')
+if not model_path.exists():
+    compressed_path = model_path.with_suffix('.tar.gz')
+    url = 'http://dl.fbaipublicfiles.com/camembert/camembert.v0.pytorch.tar.gz'
+    print('Downloading model...')
+    urllib.request.urlretrieve(url, compressed_path)
+    print('Extracting model...')
+    with tarfile.open(compressed_path) as f:
+        f.extractall(model_path.parent)
+    assert model_path.exists()
+tokenizer_path = model_path / 'sentencepiece.bpe.model'
+tokenizer = CamembertTokenizer.from_pretrained(tokenizer_path)
+model = RobertaForMaskedLM.from_pretrained(model_path)
+model.eval()
+
+masked_input = "Le camembert est <mask> :)"
+print(fill_mask(masked_input, model, tokenizer, topk=3))

From e44b939e7198fac5ce4085a1bda8e17b3934e67f Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Tue, 12 Nov 2019 17:11:00 -0800
Subject: [PATCH 171/269] Add configuration_camembert.py and
 modeling_camembert.py

---
 transformers/configuration_camembert.py |  33 +++
 transformers/modeling_camembert.py      | 257 ++++++++++++++++++++++++
 2 files changed, 290 insertions(+)
 create mode 100644 transformers/configuration_camembert.py
 create mode 100644 transformers/modeling_camembert.py

diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py
new file mode 100644
index 0000000000..07ebd6e82e
--- /dev/null
+++ b/transformers/configuration_camembert.py
@@ -0,0 +1,33 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" CamemBERT configuration """
+
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import logging
+
+from .configuration_roberta import RobertaConfig
+
+logger = logging.getLogger(__name__)
+
+CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-config.json",
+}
+
+
+class CamembertConfig(RobertaConfig):
+    pretrained_config_archive_map = CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py
new file mode 100644
index 0000000000..982c349531
--- /dev/null
+++ b/transformers/modeling_camembert.py
@@ -0,0 +1,257 @@
+# coding=utf-8
+# Copyright 2019 Inria, Facebook AI Research and the HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch CamemBERT model. """
+
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import logging
+
+from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice
+from .configuration_camembert import CamembertConfig
+from .file_utils import add_start_docstrings
+
+logger = logging.getLogger(__name__)
+
+CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
+    'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-pytorch_model.bin",
+}
+
+
+CAMEMBERT_START_DOCSTRING = r"""    The CamemBERT model was proposed in
+    `CamemBERT: a Tasty French Language Model`_
+    by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suárez, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah, and Benoît Sagot. It is based on Facebook's RoBERTa model released in 2019.
+    
+    It is a model trained on 138GB of French text.
+    
+    This implementation is the same RoBERTa.
+
+    This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and
+    refer to the PyTorch documentation for all matter related to general usage and behavior.
+
+    .. _`CamemBERT: a Tasty French Language Model`:
+        https://arxiv.org/abs/1911.03894
+
+    .. _`torch.nn.Module`:
+        https://pytorch.org/docs/stable/nn.html#module
+
+    Parameters:
+        config (:class:`~transformers.CamembertConfig`): Model configuration class with all the parameters of the 
+            model. Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
+"""
+
+CAMEMBERT_INPUTS_DOCSTRING = r"""
+    Inputs:
+        **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of input sequence tokens in the vocabulary.
+            To match pre-training, CamemBERT input sequence should be formatted with <s> and </s> tokens as follows:
+
+            (a) For sequence pairs:
+
+                ``tokens:         <s> Is this Jacksonville ? </s> </s> No it is not . </s>``
+
+            (b) For single sequences:
+
+                ``tokens:         <s> the dog is hairy . </s>``
+
+            Fully encoded sequences or sequence pairs can be obtained using the CamembertTokenizer.encode function with 
+            the ``add_special_tokens`` parameter set to ``True``.
+
+            CamemBERT is a model with absolute position embeddings so it's usually advised to pad the inputs on
+            the right rather than the left.
+
+            See :func:`transformers.PreTrainedTokenizer.encode` and
+            :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
+        **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:
+            Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
+        **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Optional segment token indices to indicate first and second portions of the inputs.
+            This embedding matrice is not trained (not pretrained during CamemBERT pretraining), you will have to train it
+            during finetuning.
+            Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1``
+            corresponds to a `sentence B` token
+            (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details).
+        **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of positions of each input sequence tokens in the position embeddings.
+            Selected in the range ``[0, config.max_position_embeddings - 1[``.
+        **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:
+            Mask to nullify selected heads of the self-attention modules.
+            Mask values selected in ``[0, 1]``:
+            ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+"""
+
+@add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.",
+                      CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING)
+class CamembertModel(RobertaModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
+            Sequence of hidden-states at the output of the last layer of the model.
+        **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)``
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            eo match pre-training, CamemBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows:
+
+            (a) For sequence pairs:
+
+                ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]``
+
+                ``token_type_ids:   0   0  0    0    0     0       0   0   0     1  1  1  1   1   1``
+
+            (b) For single sequences:
+
+                ``tokens:         [CLS] the dog is hairy . [SEP]``
+
+                ``token_type_ids:   0   0   0   0  0     0   0``
+
+            objective during Bert pretraining. This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+        model = CamembertModel.from_pretrained('camembert-base')
+        input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
+    """
+    config_class = CamembertConfig
+    pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "camembert"
+
+
+@add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """,
+    CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING)
+class CamembertForMaskedLM(RobertaForMaskedLM):
+    r"""
+        **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the masked language modeling loss.
+            Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
+            Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
+            in ``[0, ..., config.vocab_size]``
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Masked language modeling loss.
+        **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+        model = CamembertForMaskedLM.from_pretrained('camembert-base')
+        input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, masked_lm_labels=input_ids)
+        loss, prediction_scores = outputs[:2]
+
+    """
+    config_class = CamembertConfig
+    pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "camembert"
+
+
+@add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer 
+    on top of the pooled output) e.g. for GLUE tasks. """,
+    CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING)
+class CamembertForSequenceClassification(RobertaForSequenceClassification):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in ``[0, ..., config.num_labels]``.
+            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+        model = CamembertForSequenceClassification.from_pretrained('camembert-base')
+        input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+
+    """
+    config_class = CamembertConfig
+    pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "camembert"
+
+
+@add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of
+    the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """,
+    CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING)
+class CamembertForMultipleChoice(RobertaForMultipleChoice):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification loss.
+        **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension
+            of the input tensors. (see `input_ids` above).
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+        model = CamembertForMultipleChoice.from_pretrained('camembert-base')
+        choices = ["J'aime le camembert !", "Je deteste le camembert !"]
+        input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0)  # Batch size 1, 2 choices
+        labels = torch.tensor(1).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, classification_scores = outputs[:2]
+
+    """
+    config_class = CamembertConfig
+    pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "camembert"

From fb6c70a91d3183742ce0a6d97add68103253ca3a Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Tue, 12 Nov 2019 17:11:49 -0800
Subject: [PATCH 172/269] Update tokenization_camembert.py with urls

---
 transformers/tokenization_camembert.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
index 9facf7d911..0a6e751351 100644
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -11,7 +11,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License
 """ Tokenization classes for Camembert model."""
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
@@ -20,6 +20,19 @@ import sentencepiece as spm
 from transformers.tokenization_utils import PreTrainedTokenizer
 
 
+VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    'vocab_file':
+    {
+    'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-sentencepiece.bpe.model",
+    }
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    'camembert-base': None,
+}
+
 class CamembertTokenizer(PreTrainedTokenizer):
     """
         Adapted from RobertaTokenizer and XLNetTokenizer
@@ -27,7 +40,9 @@ class CamembertTokenizer(PreTrainedTokenizer):
 
             - requires `SentencePiece <https://github.com/google/sentencepiece>`_
     """
-    vocab_files_names = {'vocab_file': None}
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, vocab_file, bos_token="<s>", eos_token="</s>", sep_token="</s>",
                  cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>', **kwargs):

From f12e4d8da783a535cd8978656f0a3ca108a2e2ed Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Tue, 12 Nov 2019 17:14:50 -0800
Subject: [PATCH 173/269] Move demo_camembert.py to examples/contrib

---
 examples/{ => contrib}/demo_camembert.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename examples/{ => contrib}/demo_camembert.py (100%)

diff --git a/examples/demo_camembert.py b/examples/contrib/demo_camembert.py
similarity index 100%
rename from examples/demo_camembert.py
rename to examples/contrib/demo_camembert.py

From 3e20c2e871db82f81c3b2b814265a481be15273c Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Tue, 12 Nov 2019 17:16:24 -0800
Subject: [PATCH 174/269] Update demo_camembert.py with new classes

---
 examples/contrib/demo_camembert.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/examples/contrib/demo_camembert.py b/examples/contrib/demo_camembert.py
index df28f4f267..28144d5167 100644
--- a/examples/contrib/demo_camembert.py
+++ b/examples/contrib/demo_camembert.py
@@ -5,7 +5,7 @@ import urllib.request
 import torch
 
 from transformers.tokenization_camembert import CamembertTokenizer
-from transformers.modeling_roberta import RobertaForMaskedLM
+from transformers.modeling_camembert import CamembertForMaskedLM
 
 
 def fill_mask(masked_input, model, tokenizer, topk=5):
@@ -40,19 +40,8 @@ def fill_mask(masked_input, model, tokenizer, topk=5):
     return topk_filled_outputs
 
 
-model_path = Path('camembert.v0.pytorch')
-if not model_path.exists():
-    compressed_path = model_path.with_suffix('.tar.gz')
-    url = 'http://dl.fbaipublicfiles.com/camembert/camembert.v0.pytorch.tar.gz'
-    print('Downloading model...')
-    urllib.request.urlretrieve(url, compressed_path)
-    print('Extracting model...')
-    with tarfile.open(compressed_path) as f:
-        f.extractall(model_path.parent)
-    assert model_path.exists()
-tokenizer_path = model_path / 'sentencepiece.bpe.model'
-tokenizer = CamembertTokenizer.from_pretrained(tokenizer_path)
-model = RobertaForMaskedLM.from_pretrained(model_path)
+tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+model = CamembertForMaskedLM.from_pretrained('camembert-base')
 model.eval()
 
 masked_input = "Le camembert est <mask> :)"

From 694d4fcbb61f15b66781219954112791248d832e Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Tue, 12 Nov 2019 17:28:37 -0800
Subject: [PATCH 175/269] Add CamemBERT classes to __init__.py

---
 transformers/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index dd9e18a050..cdf0669b39 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE
 from .tokenization_xlm import XLMTokenizer
 from .tokenization_roberta import RobertaTokenizer
 from .tokenization_distilbert import DistilBertTokenizer
+from .tokenization_camembert import CamembertTokenizer
 
 # Configurations
 from .configuration_utils import PretrainedConfig
@@ -56,6 +57,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
 
 # Modeling
 if is_torch_available():
@@ -96,6 +98,9 @@ if is_torch_available():
                                 DistilBertForSequenceClassification, DistilBertForQuestionAnswering,
                                 DistilBertForTokenClassification,
                                 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+    from .modeling_camembert import (CamembertForMaskedLM, CamembertModel,
+                                CamembertForSequenceClassification, CamembertForMultipleChoice,
+                                CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
     # Optimization

From 035fea53157b43d311b1e1164395398c4ec0dda5 Mon Sep 17 00:00:00 2001
From: Louis MARTIN <louisrtm@gmail.com>
Date: Tue, 12 Nov 2019 17:41:41 -0800
Subject: [PATCH 176/269] Add CamemBERT to auto files and docs

---
 docs/source/pretrained_models.rst  | 6 +++++-
 transformers/configuration_auto.py | 7 ++++++-
 transformers/tokenization_auto.py  | 9 +++++++--
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index edb47e7f1c..b0a578fd80 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -155,5 +155,9 @@ Here is the full list of the currently provided pretrained models together with
 | CTRL              | ``ctrl``                                                   | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters                                                                                    |
 |                   |                                                            | | Salesforce's Large-sized CTRL English model                                                                                         |
 +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+| CamemBERT         | ``camembert-base``                                         | | 12-layer, 768-hidden, 12-heads, 110M parameters                                                                                     |
+|                   |                                                            | | CamemBERT using the BERT-base architecture                                                                                          |
+|                   |                                                            | (see `details <https://github.com/pytorch/fairseq/tree/master/examples/camembert>`__)                                                 |
++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 
-.. <https://huggingface.co/transformers/examples.html>`__
\ No newline at end of file
+.. <https://huggingface.co/transformers/examples.html>`__
diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py
index edd21a670c..2906136139 100644
--- a/transformers/configuration_auto.py
+++ b/transformers/configuration_auto.py
@@ -27,6 +27,7 @@ from .configuration_xlm import XLMConfig
 from .configuration_roberta import RobertaConfig
 from .configuration_distilbert import DistilBertConfig
 from .configuration_ctrl import CTRLConfig
+from .configuration_camembert import CamembertConfig
 
 logger = logging.getLogger(__name__)
 
@@ -50,6 +51,7 @@ class AutoConfig(object):
             - contains `xlnet`: XLNetConfig (XLNet model)
             - contains `xlm`: XLMConfig (XLM model)
             - contains `roberta`: RobertaConfig (RoBERTa model)
+            - contains `camembert`: CamembertConfig (CamemBERT model)
             - contains `ctrl` : CTRLConfig (CTRL model)
         This class cannot be instantiated using `__init__()` (throw an error).
     """
@@ -72,6 +74,7 @@ class AutoConfig(object):
             - contains `xlnet`: XLNetConfig (XLNet model)
             - contains `xlm`: XLMConfig (XLM model)
             - contains `roberta`: RobertaConfig (RoBERTa model)
+            - contains `camembert`: CamembertConfig (CamemBERT model)
             - contains `ctrl` : CTRLConfig (CTRL model)
         Params:
             pretrained_model_name_or_path: either:
@@ -116,6 +119,8 @@ class AutoConfig(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+        elif 'camembert' in pretrained_model_name_or_path:
+            return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
         elif 'bert' in pretrained_model_name_or_path:
@@ -134,4 +139,4 @@ class AutoConfig(object):
             return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
         raise ValueError("Unrecognized model identifier in {}. Should contains one of "
                          "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-                         "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path))
+                         "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path))
diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py
index ec056de17f..4510159905 100644
--- a/transformers/tokenization_auto.py
+++ b/transformers/tokenization_auto.py
@@ -27,6 +27,7 @@ from .tokenization_xlnet import XLNetTokenizer
 from .tokenization_xlm import XLMTokenizer
 from .tokenization_roberta import RobertaTokenizer
 from .tokenization_distilbert import DistilBertTokenizer
+from .tokenization_camembert import CamembertTokenizer
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +42,7 @@ class AutoTokenizer(object):
 
         The tokenizer class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
+            - contains `camembert`: CamembertTokenizer (CamemBERT model)
             - contains `distilbert`: DistilBertTokenizer (DistilBert model)
             - contains `roberta`: RobertaTokenizer (RoBERTa model)
             - contains `bert`: BertTokenizer (Bert model)
@@ -64,8 +66,9 @@ class AutoTokenizer(object):
 
         The tokenizer class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
+            - contains `camembert`: CamembertTokenizer (CamemBERT model)
             - contains `distilbert`: DistilBertTokenizer (DistilBert model)
-            - contains `roberta`: RobertaTokenizer (XLM model)
+            - contains `roberta`: RobertaTokenizer (RoBERTa model)
             - contains `bert`: BertTokenizer (Bert model)
             - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model)
             - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model)
@@ -103,6 +106,8 @@ class AutoTokenizer(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+        elif 'camembert' in pretrained_model_name_or_path:
+            return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
         elif 'bert' in pretrained_model_name_or_path:
@@ -121,4 +126,4 @@ class AutoTokenizer(object):
             return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
         raise ValueError("Unrecognized model identifier in {}. Should contains one of "
                          "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-                         "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path))
+                         "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path))

From 26858f27cb352b6bb1cda2b090413d1d2206a9ee Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Fri, 15 Nov 2019 23:23:31 -0500
Subject: [PATCH 177/269] [camembert] Upload to s3 + rename script

---
 examples/contrib/{demo_camembert.py => run_camembert.py} | 0
 transformers/configuration_camembert.py                  | 2 +-
 transformers/modeling_camembert.py                       | 2 +-
 transformers/tokenization_camembert.py                   | 2 +-
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename examples/contrib/{demo_camembert.py => run_camembert.py} (100%)

diff --git a/examples/contrib/demo_camembert.py b/examples/contrib/run_camembert.py
similarity index 100%
rename from examples/contrib/demo_camembert.py
rename to examples/contrib/run_camembert.py
diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py
index 07ebd6e82e..3ff64454e5 100644
--- a/transformers/configuration_camembert.py
+++ b/transformers/configuration_camembert.py
@@ -25,7 +25,7 @@ from .configuration_roberta import RobertaConfig
 logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-config.json",
+    'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json",
 }
 
 
diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py
index 982c349531..a0f5933bc2 100644
--- a/transformers/modeling_camembert.py
+++ b/transformers/modeling_camembert.py
@@ -27,7 +27,7 @@ from .file_utils import add_start_docstrings
 logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-pytorch_model.bin",
+    'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin",
 }
 
 
diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
index 0a6e751351..de587ac863 100644
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -25,7 +25,7 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}
 PRETRAINED_VOCAB_FILES_MAP = {
     'vocab_file':
     {
-    'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-sentencepiece.bpe.model",
+    'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model",
     }
 }
 

From f9abf73e319d99ed74f46671e98b00b9328cd245 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Fri, 15 Nov 2019 23:40:20 -0500
Subject: [PATCH 178/269] [camembert] realign w/ recent changes

---
 transformers/modeling_camembert.py     | 10 +++++-----
 transformers/tokenization_camembert.py |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py
index a0f5933bc2..05538926e2 100644
--- a/transformers/modeling_camembert.py
+++ b/transformers/modeling_camembert.py
@@ -37,7 +37,7 @@ CAMEMBERT_START_DOCSTRING = r"""    The CamemBERT model was proposed in
     
     It is a model trained on 138GB of French text.
     
-    This implementation is the same RoBERTa.
+    This implementation is the same as RoBERTa.
 
     This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and
     refer to the PyTorch documentation for all matter related to general usage and behavior.
@@ -94,6 +94,10 @@ CAMEMBERT_INPUTS_DOCSTRING = r"""
             Mask to nullify selected heads of the self-attention modules.
             Mask values selected in ``[0, 1]``:
             ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+        **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:
+            Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+            than the model's internal embedding lookup matrix.
 """
 
 @add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.",
@@ -143,7 +147,6 @@ class CamembertModel(RobertaModel):
     """
     config_class = CamembertConfig
     pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
-    base_model_prefix = "camembert"
 
 
 @add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """,
@@ -180,7 +183,6 @@ class CamembertForMaskedLM(RobertaForMaskedLM):
     """
     config_class = CamembertConfig
     pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
-    base_model_prefix = "camembert"
 
 
 @add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer 
@@ -219,7 +221,6 @@ class CamembertForSequenceClassification(RobertaForSequenceClassification):
     """
     config_class = CamembertConfig
     pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
-    base_model_prefix = "camembert"
 
 
 @add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of
@@ -254,4 +255,3 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice):
     """
     config_class = CamembertConfig
     pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
-    base_model_prefix = "camembert"
diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
index de587ac863..ae1b322941 100644
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -87,7 +87,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
                 special tokens for the model
 
         Returns:
-            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
         if already_has_special_tokens:
             if token_ids_1 is not None:

From 0477b307c7501ea76e01b03cb387a2312db752b3 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Fri, 15 Nov 2019 23:54:11 -0500
Subject: [PATCH 179/269] [camembert] tokenizer: use additional_special_tokens

---
 transformers/tokenization_camembert.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
index ae1b322941..41d3d74cff 100644
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -45,10 +45,12 @@ class CamembertTokenizer(PreTrainedTokenizer):
     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, vocab_file, bos_token="<s>", eos_token="</s>", sep_token="</s>",
-                 cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>', **kwargs):
+                 cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>',
+                 additional_special_tokens=['<s>NOTUSED', '<s>NOTUSED'], **kwargs):
         super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
                                                  sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
-                                                 mask_token=mask_token, **kwargs)
+                                                 mask_token=mask_token, additional_special_tokens=additional_special_tokens,
+                                                 **kwargs)
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens
         self.sp_model = spm.SentencePieceProcessor()

From d32ce2c8df7053c19061b709465cdcc765e45a15 Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 18 Nov 2019 14:14:19 +0100
Subject: [PATCH 180/269] camembert: add wrapper for
 CamembertForTokenClassification

---
 transformers/modeling_camembert.py | 38 +++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py
index 05538926e2..f302346f2d 100644
--- a/transformers/modeling_camembert.py
+++ b/transformers/modeling_camembert.py
@@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function,
 
 import logging
 
-from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice
+from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification
 from .configuration_camembert import CamembertConfig
 from .file_utils import add_start_docstrings
 
@@ -255,3 +255,39 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice):
     """
     config_class = CamembertConfig
     pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+
+
+@add_start_docstrings("""CamemBERT Model with a token classification head on top (a linear layer on top of
+    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,
+    CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING)
+class CamembertForTokenClassification(RobertaForTokenClassification):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the token classification loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification loss.
+        **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)``
+            Classification scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+        model = CamembertForTokenClassification.from_pretrained('camembert-base')
+        input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, scores = outputs[:2]
+
+    """
+    config_class = CamembertConfig
+    pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP

From 33753d9139307d9635db0309b6ddb9c53192c60a Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 18 Nov 2019 14:14:54 +0100
Subject: [PATCH 181/269] module: import CamembertForTokenClassification

---
 transformers/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index cdf0669b39..5c7b0a6197 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -100,6 +100,7 @@ if is_torch_available():
                                 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_camembert import (CamembertForMaskedLM, CamembertModel,
                                 CamembertForSequenceClassification, CamembertForMultipleChoice,
+                                CamembertForTokenClassification,
                                 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 

From 44455eb5b61b2fc4a0bdbea6775d328c45a597e5 Mon Sep 17 00:00:00 2001
From: Sebastian Stabinger <sebastian@stabinger.name>
Date: Mon, 18 Nov 2019 10:07:05 +0100
Subject: [PATCH 182/269] Adds CamemBERT to Model architectures list

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 40d6acb76e..d1b37dbd48 100644
--- a/README.md
+++ b/README.md
@@ -122,7 +122,8 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation).
 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
-10. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
+10. **[CamemBERT](https://camembert-model.fr)** (from Facebook, Inria, Sorbonne) French language model based on RoBERTa released together with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Martin, Muller, and Suarez et al. 
+11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
 

From 3916b334a86484af8442d1cfdb2f15695feae581 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Mon, 18 Nov 2019 09:29:11 -0500
Subject: [PATCH 183/269] [camembert] Acknowledge the full author list

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d1b37dbd48..a49e8086d1 100644
--- a/README.md
+++ b/README.md
@@ -122,7 +122,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation).
 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
-10. **[CamemBERT](https://camembert-model.fr)** (from Facebook, Inria, Sorbonne) French language model based on RoBERTa released together with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Martin, Muller, and Suarez et al. 
+10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).

From 0b3d45eb64607158977f546d57f90eae268c7836 Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 18 Nov 2019 15:49:44 +0100
Subject: [PATCH 184/269] camembert: add implementation for save_vocabulary
 method

---
 transformers/tokenization_camembert.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py
index 41d3d74cff..bf2a6fe993 100644
--- a/transformers/tokenization_camembert.py
+++ b/transformers/tokenization_camembert.py
@@ -16,9 +16,14 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
+import logging
+import os
+from shutil import copyfile
+
 import sentencepiece as spm
 from transformers.tokenization_utils import PreTrainedTokenizer
 
+logger = logging.getLogger(__name__)
 
 VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}
 
@@ -55,6 +60,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(str(vocab_file))
+        self.vocab_file = vocab_file
         # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
         # sentencepiece vocabulary (this is the case for <s> and </s>
         self.fairseq_tokens_to_ids = {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
@@ -135,3 +141,17 @@ class CamembertTokenizer(PreTrainedTokenizer):
         if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset)
+
+    def save_vocabulary(self, save_directory):
+        """ Save the sentencepiece vocabulary (copy original file) and special tokens file
+            to a directory.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
+            return
+        out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+
+        return (out_vocab_file,)

From 56c84863a1a20dfb82b928c5c9f77c21d9def8c7 Mon Sep 17 00:00:00 2001
From: Stefan Schweter <stefan.schweter@bsb-muenchen.de>
Date: Mon, 18 Nov 2019 15:50:16 +0100
Subject: [PATCH 185/269] camembert: add support for CamemBERT in run_ner
 example

---
 examples/run_ner.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index 4359e587ae..127d63a6cd 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -37,6 +37,7 @@ from transformers import AdamW, get_linear_schedule_with_warmup
 from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer
 from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer
 from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer
+from transformers import CamembertConfig, CamembertForTokenClassification, CamembertTokenizer
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +48,8 @@ ALL_MODELS = sum(
 MODEL_CLASSES = {
     "bert": (BertConfig, BertForTokenClassification, BertTokenizer),
     "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer)
+    "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer),
+    "camembert": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer),
 }
 
 

From f3386d938348628c91457fc7d8650c223317a053 Mon Sep 17 00:00:00 2001
From: Kazutoshi Shinoda <kazutoshi.shinoda0516@gmail.com>
Date: Sun, 17 Nov 2019 18:08:51 +0900
Subject: [PATCH 186/269] typo "deay" -> "decay"

---
 examples/run_squad.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/run_squad.py b/examples/run_squad.py
index d7fdc32ae7..69088d73c3 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -402,7 +402,7 @@ def main():
     parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
                         help="Number of updates steps to accumulate before performing a backward/update pass.")
     parser.add_argument("--weight_decay", default=0.0, type=float,
-                        help="Weight deay if we apply some.")
+                        help="Weight decay if we apply some.")
     parser.add_argument("--adam_epsilon", default=1e-8, type=float,
                         help="Epsilon for Adam optimizer.")
     parser.add_argument("--max_grad_norm", default=1.0, type=float,

From 454455c695ff38df1ed3670a43677fdd1abcedf3 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 20 Nov 2019 09:42:48 -0500
Subject: [PATCH 187/269] fix #1879

---
 examples/utils_squad.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/utils_squad.py b/examples/utils_squad.py
index c04dacf6d3..4f1c581588 100644
--- a/examples/utils_squad.py
+++ b/examples/utils_squad.py
@@ -240,6 +240,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
 
         # The -3 accounts for [CLS], [SEP] and [SEP]
         max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
+        assert max_tokens_for_doc > 0
 
         # We can have documents that are longer than the maximum sequence length.
         # To deal with this we do a sliding window approach, where we take chunks

From e70cdf083ddb8bfe298d43e6d70d698a3a2f56d3 Mon Sep 17 00:00:00 2001
From: Jin Young Sohn <jysohn@google.com>
Date: Wed, 20 Nov 2019 22:33:26 +0000
Subject: [PATCH 188/269] Cleanup TPU bits from run_glue.py

TPU runner is currently implemented in:
https://github.com/pytorch-tpu/transformers/blob/tpu/examples/run_glue_tpu.py.

We plan to upstream this directly into `huggingface/transformers`
(either `master` or `tpu`) branch once it's been more thoroughly tested.
---
 examples/run_glue.py | 35 ++---------------------------------
 1 file changed, 2 insertions(+), 33 deletions(-)

diff --git a/examples/run_glue.py b/examples/run_glue.py
index 19316cb0ec..527e440075 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -158,7 +158,7 @@ def train(args, train_dataset, model, tokenizer):
                 loss.backward()
 
             tr_loss += loss.item()
-            if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu:
+            if (step + 1) % args.gradient_accumulation_steps == 0:
                 if args.fp16:
                     torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
                 else:
@@ -189,11 +189,6 @@ def train(args, train_dataset, model, tokenizer):
                     torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                     logger.info("Saving model checkpoint to %s", output_dir)
 
-            if args.tpu:
-                args.xla_model.optimizer_step(optimizer, barrier=True)
-                model.zero_grad()
-                global_step += 1
-
             if args.max_steps > 0 and global_step > args.max_steps:
                 epoch_iterator.close()
                 break
@@ -397,15 +392,6 @@ def main():
     parser.add_argument('--seed', type=int, default=42,
                         help="random seed for initialization")
 
-    parser.add_argument('--tpu', action='store_true',
-                        help="Whether to run on the TPU defined in the environment variables")
-    parser.add_argument('--tpu_ip_address', type=str, default='',
-                        help="TPU IP address if none are set in the environment variables")
-    parser.add_argument('--tpu_name', type=str, default='',
-                        help="TPU name if none are set in the environment variables")
-    parser.add_argument('--xrt_tpu_config', type=str, default='',
-                        help="XRT TPU config if none are set in the environment variables")
-
     parser.add_argument('--fp16', action='store_true',
                         help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
     parser.add_argument('--fp16_opt_level', type=str, default='O1',
@@ -439,23 +425,6 @@ def main():
         args.n_gpu = 1
     args.device = device
 
-    if args.tpu:
-        if args.tpu_ip_address:
-            os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address
-        if args.tpu_name:
-            os.environ["TPU_NAME"] = args.tpu_name
-        if args.xrt_tpu_config:
-            os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config
-
-        assert "TPU_IP_ADDRESS" in os.environ
-        assert "TPU_NAME" in os.environ
-        assert "XRT_TPU_CONFIG" in os.environ
-
-        import torch_xla
-        import torch_xla.core.xla_model as xm
-        args.device = xm.xla_device()
-        args.xla_model = xm
-
     # Setup logging
     logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                         datefmt = '%m/%d/%Y %H:%M:%S',
@@ -509,7 +478,7 @@ def main():
 
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
-    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu:
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
         # Create output directory if needed
         if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
             os.makedirs(args.output_dir)

From 6f70bb8c69cdbe1207e4909ccface32c8f51297b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 20 Nov 2019 18:01:03 +0100
Subject: [PATCH 189/269] add instructions to run the examples

---
 README.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/README.md b/README.md
index a49e8086d1..9cc783fd71 100644
--- a/README.md
+++ b/README.md
@@ -86,6 +86,18 @@ When TensorFlow 2.0 and/or PyTorch has been installed, you can install from sour
 pip install [--editable] .
 ```
 
+### Run the examples
+
+Examples are included in the repository but are not shipped with the library.
+Therefore, in order to run the examples you will first need to clone the
+repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps:
+
+```bash
+git clone git@github.com:huggingface/transformers
+cd transformers
+pip install .
+```
+
 ### Tests
 
 A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
@@ -253,6 +265,11 @@ print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sen
 
 ## Quick tour of the fine-tuning/usage scripts
 
+**Important**  
+Before running the fine-tuning scripts, please read the
+[instructions](#run-the-examples) on how to
+setup your environment to run the examples.
+
 The library comprises several example scripts with SOTA performances for NLU and NLG tasks:
 
 - `run_glue.py`: an example fine-tuning Bert, XLNet and XLM on nine different GLUE tasks (*sequence-level classification*)

From 26db31e0c09a8b5e1ca7a61c454b159eab9d86be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Wed, 20 Nov 2019 18:13:38 +0100
Subject: [PATCH 190/269] update the documentation

---
 examples/README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/examples/README.md b/examples/README.md
index abb4cb6e5a..622fa07f8f 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -3,6 +3,15 @@
 In this section a few examples are put together. All of these examples work for several models, making use of the very
 similar API between the different models.
 
+**Important**  
+To use the examples, execute the following steps in a new virtual environment:
+
+```bash
+git clone git@github.com:huggingface/transformers
+cd transformers
+pip install .
+```
+
 | Section                    | Description                                                                                                                                                |
 |----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [TensorFlow 2.0 models on GLUE](#TensorFlow-2.0-Bert-models-on-GLUE) | Examples running BERT TensorFlow 2.0 model on the GLUE tasks. 

From 041a901f324eea7e7ee04b0f7a563c7ed5c8a03a Mon Sep 17 00:00:00 2001
From: Nikolay Korolev <korolevns98@gmail.com>
Date: Fri, 22 Nov 2019 22:56:43 +0300
Subject: [PATCH 191/269] Fix typo in documentation. toto -> to

---
 transformers/tokenization_gpt2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py
index 4bec515903..a4798b0129 100644
--- a/transformers/tokenization_gpt2.py
+++ b/transformers/tokenization_gpt2.py
@@ -184,7 +184,7 @@ class GPT2Tokenizer(PreTrainedTokenizer):
         """ Tokenize a string.
             Args:
                 - add_prefix_space (boolean, default False):
-                    Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers.
+                    Begin the sentence with at least one space to get invariance to word order in GPT-2 (and RoBERTa) tokenizers.
         """
         if add_prefix_space:
             text = ' ' + text

From 176cd1ce1b337134425b426207fbe155099c18b4 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Sat, 23 Nov 2019 11:18:54 -0500
Subject: [PATCH 192/269] [doc] homogenize instructions slightly

---
 README.md          | 5 ++---
 examples/README.md | 4 ++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 9cc783fd71..52e74a6f80 100644
--- a/README.md
+++ b/README.md
@@ -89,13 +89,12 @@ pip install [--editable] .
 ### Run the examples
 
 Examples are included in the repository but are not shipped with the library.
-Therefore, in order to run the examples you will first need to clone the
-repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps:
+Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps:
 
 ```bash
 git clone git@github.com:huggingface/transformers
 cd transformers
-pip install .
+pip install [--editable] .
 ```
 
 ### Tests
diff --git a/examples/README.md b/examples/README.md
index 622fa07f8f..0c57990ea7 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -4,12 +4,12 @@ In this section a few examples are put together. All of these examples work for
 similar API between the different models.
 
 **Important**  
-To use the examples, execute the following steps in a new virtual environment:
+To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment:
 
 ```bash
 git clone git@github.com:huggingface/transformers
 cd transformers
-pip install .
+pip install [--editable] .
 ```
 
 | Section                    | Description                                                                                                                                                |

From afaa33585109550f9ecaaee4e47f187aaaefedd0 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Sat, 23 Nov 2019 11:34:45 -0500
Subject: [PATCH 193/269] [doc] Fix assets urls

---
 docs/source/_static/js/custom.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js
index 2c7836fd20..ec804b3704 100644
--- a/docs/source/_static/js/custom.js
+++ b/docs/source/_static/js/custom.js
@@ -1,5 +1,5 @@
 function addIcon() {
-    const huggingFaceLogo = "https://huggingface.co/assets/transformers-docs/huggingface_logo.svg";
+    const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg";
     const image = document.createElement("img");
     image.setAttribute("src", huggingFaceLogo);
 
@@ -24,10 +24,10 @@ function addCustomFooter() {
     social.classList.add("footer__Social");
 
     const imageDetails = [
-        { link: "https://huggingface.co", imageLink: "https://huggingface.co/assets/transformers-docs/website.svg" },
-        { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/twitter.svg" },
-        { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/github.svg" },
-        { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/assets/transformers-docs/linkedin.svg" }
+        { link: "https://huggingface.co", imageLink: "https://huggingface.co/landing/assets/transformers-docs/website.svg" },
+        { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/twitter.svg" },
+        { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/github.svg" },
+        { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/landing/assets/transformers-docs/linkedin.svg" }
     ];
 
     imageDetails.forEach(imageLinks => {

From 7485caefb09cb7f4c4b720b40ec69fed345a6b1c Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Mon, 25 Nov 2019 09:33:39 -0500
Subject: [PATCH 194/269] fix #1894

---
 examples/run_lm_finetuning.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index 52a1b75a65..aded521c1d 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -68,7 +68,7 @@ class TextDataset(Dataset):
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename)
 
-        if os.path.exists(cached_features_file):
+        if os.path.exists(cached_features_file) and not args.overwrite_cache:
             logger.info("Loading features from cached file %s", cached_features_file)
             with open(cached_features_file, 'rb') as handle:
                 self.examples = pickle.load(handle)

From 99f750d64e78d20fa5213ea11235b6a1b084481e Mon Sep 17 00:00:00 2001
From: Evpok Padding <evpok.padding@gmail.com>
Date: Thu, 21 Nov 2019 10:35:07 +0100
Subject: [PATCH 195/269] add Camembert models to modeling_auto

---
 transformers/modeling_auto.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py
index d98110d4bd..a5828148ad 100644
--- a/transformers/modeling_auto.py
+++ b/transformers/modeling_auto.py
@@ -27,6 +27,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi
 from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering
 from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification
 from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification
+from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice
 
 from .modeling_utils import PreTrainedModel, SequenceSummary
 
@@ -48,6 +49,7 @@ class AutoModel(object):
         The base model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertModel (DistilBERT model)
+            - contains `camembert`: CamemBERTModel (CamemBERT model)
             - contains `roberta`: RobertaModel (RoBERTa model)
             - contains `bert`: BertModel (Bert model)
             - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
@@ -71,6 +73,7 @@ class AutoModel(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertModel (DistilBERT model)
+            - contains `camembert`: CamemBERTModel (CamemBERT model)
             - contains `roberta`: RobertaModel (RoBERTa model)
             - contains `bert`: BertModel (Bert model)
             - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
@@ -138,6 +141,8 @@ class AutoModel(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        elif 'camembert' in pretrained_model_name_or_path:
+            return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'bert' in pretrained_model_name_or_path:
@@ -172,6 +177,7 @@ class AutoModelWithLMHead(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model)
+            - contains `camembert`: CamemBERTModel (CamemBERT model)
             - contains `roberta`: RobertaForMaskedLM (RoBERTa model)
             - contains `bert`: BertForMaskedLM (Bert model)
             - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model)
@@ -198,6 +204,7 @@ class AutoModelWithLMHead(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model)
+            - contains `camembert`: CamemBERTModel (CamemBERT model)
             - contains `roberta`: RobertaForMaskedLM (RoBERTa model)
             - contains `bert`: BertForMaskedLM (Bert model)
             - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model)
@@ -264,6 +271,8 @@ class AutoModelWithLMHead(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        if 'camembert' in pretrained_model_name_or_path:
+            return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'bert' in pretrained_model_name_or_path:
@@ -298,6 +307,7 @@ class AutoModelForSequenceClassification(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model)
+            - contains `camembert`: CamemBERTModel (CamemBERT model)
             - contains `roberta`: RobertaForSequenceClassification (RoBERTa model)
             - contains `bert`: BertForSequenceClassification (Bert model)
             - contains `xlnet`: XLNetForSequenceClassification (XLNet model)
@@ -320,6 +330,7 @@ class AutoModelForSequenceClassification(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model)
+            - contains `camembert`: CamemBERTModel (CamemBERT model)
             - contains `roberta`: RobertaForSequenceClassification (RoBERTa model)
             - contains `bert`: BertForSequenceClassification (Bert model)
             - contains `xlnet`: XLNetForSequenceClassification (XLNet model)
@@ -383,6 +394,8 @@ class AutoModelForSequenceClassification(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        if 'camembert' in pretrained_model_name_or_path:
+            return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'bert' in pretrained_model_name_or_path:

From c8eb8157b86291413fe8096217f4defc168aa73f Mon Sep 17 00:00:00 2001
From: Evpok Padding <evpok.padding@gmail.com>
Date: Thu, 21 Nov 2019 11:01:20 +0100
Subject: [PATCH 196/269] fix docstrings

---
 transformers/modeling_auto.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py
index a5828148ad..ce36f6dc4a 100644
--- a/transformers/modeling_auto.py
+++ b/transformers/modeling_auto.py
@@ -49,7 +49,7 @@ class AutoModel(object):
         The base model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertModel (DistilBERT model)
-            - contains `camembert`: CamemBERTModel (CamemBERT model)
+            - contains `camembert`: CamembertModel (CamemBERT model)
             - contains `roberta`: RobertaModel (RoBERTa model)
             - contains `bert`: BertModel (Bert model)
             - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
@@ -73,7 +73,7 @@ class AutoModel(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertModel (DistilBERT model)
-            - contains `camembert`: CamemBERTModel (CamemBERT model)
+            - contains `camembert`: CamembertModel (CamemBERT model)
             - contains `roberta`: RobertaModel (RoBERTa model)
             - contains `bert`: BertModel (Bert model)
             - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model)
@@ -177,7 +177,7 @@ class AutoModelWithLMHead(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model)
-            - contains `camembert`: CamemBERTModel (CamemBERT model)
+            - contains `camembert`: CamembertForMaskedLM (CamemBERT model)
             - contains `roberta`: RobertaForMaskedLM (RoBERTa model)
             - contains `bert`: BertForMaskedLM (Bert model)
             - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model)
@@ -204,7 +204,7 @@ class AutoModelWithLMHead(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model)
-            - contains `camembert`: CamemBERTModel (CamemBERT model)
+            - contains `camembert`: CamembertForMaskedLM (CamemBERT model)
             - contains `roberta`: RobertaForMaskedLM (RoBERTa model)
             - contains `bert`: BertForMaskedLM (Bert model)
             - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model)
@@ -307,7 +307,7 @@ class AutoModelForSequenceClassification(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model)
-            - contains `camembert`: CamemBERTModel (CamemBERT model)
+            - contains `camembert`: CamembertForSequenceClassification (CamemBERT model)
             - contains `roberta`: RobertaForSequenceClassification (RoBERTa model)
             - contains `bert`: BertForSequenceClassification (Bert model)
             - contains `xlnet`: XLNetForSequenceClassification (XLNet model)
@@ -330,7 +330,7 @@ class AutoModelForSequenceClassification(object):
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
             - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model)
-            - contains `camembert`: CamemBERTModel (CamemBERT model)
+            - contains `camembert`: CamembertForSequenceClassification (CamemBERT model)
             - contains `roberta`: RobertaForSequenceClassification (RoBERTa model)
             - contains `bert`: BertForSequenceClassification (Bert model)
             - contains `xlnet`: XLNetForSequenceClassification (XLNet model)

From fa963ecc59a1dea59c2d0e952b2c4483e1828176 Mon Sep 17 00:00:00 2001
From: Evpok Padding <evpok.padding@gmail.com>
Date: Thu, 21 Nov 2019 11:03:12 +0100
Subject: [PATCH 197/269] =?UTF-8?q?if=E2=86=92elif?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 transformers/modeling_auto.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py
index ce36f6dc4a..5866420001 100644
--- a/transformers/modeling_auto.py
+++ b/transformers/modeling_auto.py
@@ -271,7 +271,7 @@ class AutoModelWithLMHead(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        if 'camembert' in pretrained_model_name_or_path:
+        elif 'camembert' in pretrained_model_name_or_path:
             return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
@@ -394,7 +394,7 @@ class AutoModelForSequenceClassification(object):
         """
         if 'distilbert' in pretrained_model_name_or_path:
             return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        if 'camembert' in pretrained_model_name_or_path:
+        elif 'camembert' in pretrained_model_name_or_path:
             return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
         elif 'roberta' in pretrained_model_name_or_path:
             return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)

From 07bf43074f39379d8c7f6afcca105e69685f7531 Mon Sep 17 00:00:00 2001
From: Bilal Khan <bk@tinymanager.com>
Date: Thu, 21 Nov 2019 20:52:06 -0600
Subject: [PATCH 198/269] Fix GPT2 docstring

---
 transformers/tokenization_gpt2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py
index a4798b0129..5fda709448 100644
--- a/transformers/tokenization_gpt2.py
+++ b/transformers/tokenization_gpt2.py
@@ -107,10 +107,10 @@ class GPT2Tokenizer(PreTrainedTokenizer):
     """
     GPT-2 BPE tokenizer. Peculiarities:
         - Byte-level Byte-Pair-Encoding
-        - Requires a space to start the input string => the encoding methods should be called with the
+        - Requires a space to start the input string => the encoding and tokenize methods should be called with the
           ``add_prefix_space`` flag set to ``True``.
-          Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve
-          the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"`
+          Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve
+          the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(" Hello")) = "Hello"`
     """
     vocab_files_names = VOCAB_FILES_NAMES
     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP

From aa92a184d2b92faadec975139ad55e2ae749362c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=B0brahim=20Ethem=20Demirci?= <dmrc_wr@hotmail.com>
Date: Sat, 16 Nov 2019 16:49:37 +0300
Subject: [PATCH 199/269] resize model when special tokenizer present

---
 examples/run_lm_finetuning.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py
index aded521c1d..c33aa94a32 100644
--- a/examples/run_lm_finetuning.py
+++ b/examples/run_lm_finetuning.py
@@ -215,6 +215,7 @@ def train(args, train_dataset, model, tokenizer):
 
     global_step = 0
     tr_loss, logging_loss = 0.0, 0.0
+    model.resize_token_embeddings(len(tokenizer))
     model.zero_grad()
     train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
     set_seed(args)  # Added here for reproducibility (even between python 2 and 3)

From 5d3b8daad2cc6287d30f03f8a96d0a1f7bc8d0dc Mon Sep 17 00:00:00 2001
From: manansanghi <52307004+manansanghi@users.noreply.github.com>
Date: Fri, 22 Nov 2019 10:31:54 -0800
Subject: [PATCH 200/269] Minor bug fixes on run_ner.py

---
 examples/run_ner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/run_ner.py b/examples/run_ner.py
index 127d63a6cd..1ab1236d94 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -127,7 +127,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id):
                       "attention_mask": batch[1],
                       "labels": batch[3]}
             if args.model_type != "distilbert":
-                inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None  # XLM and RoBERTa don"t use segment_ids
+                inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None  # XLM and RoBERTa don"t use segment_ids
 
             outputs = model(**inputs)
             loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)
@@ -217,7 +217,7 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=""
                       "attention_mask": batch[1],
                       "labels": batch[3]}
             if args.model_type != "distilbert":
-                inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None  # XLM and RoBERTa don"t use segment_ids
+                inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None  # XLM and RoBERTa don"t use segment_ids
             outputs = model(**inputs)
             tmp_eval_loss, logits = outputs[:2]
 

From 8e5d84fcc1a645d3c13b8a2f64fa995637440dad Mon Sep 17 00:00:00 2001
From: v_sboliu <v_sboliu@tencent.com>
Date: Tue, 26 Nov 2019 12:04:31 +0800
Subject: [PATCH 201/269] Fixed typo

---
 transformers/modeling_bert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py
index 7c2c6f4602..81d92d8f1b 100644
--- a/transformers/modeling_bert.py
+++ b/transformers/modeling_bert.py
@@ -278,7 +278,7 @@ class BertAttention(nn.Module):
         if len(heads) == 0:
             return
         mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size)
-        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
+        heads = set(heads) - self.pruned_heads  # Convert to set and remove already pruned heads
         for head in heads:
             # Compute how many pruned heads are before the head and move the index accordingly
             head = head - sum(1 if h < head else 0 for h in self.pruned_heads)

From c0c2088333e2e8ce2b24d0c7f4bf071dcccbd7ea Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 29 Oct 2019 19:57:38 +0000
Subject: [PATCH 202/269] ALBERT model

---
 transformers/configuration_albert.py          |  72 ++++
 ...lbert_original_tf_checkpoint_to_pytorch.py |  44 +++
 transformers/modeling_albert.py               | 331 ++++++++++++++++++
 3 files changed, 447 insertions(+)
 create mode 100644 transformers/configuration_albert.py
 create mode 100644 transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
 create mode 100644 transformers/modeling_albert.py

diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py
new file mode 100644
index 0000000000..c86c9565cb
--- /dev/null
+++ b/transformers/configuration_albert.py
@@ -0,0 +1,72 @@
+from .configuration_utils import PretrainedConfig
+
+class AlbertConfig(PretrainedConfig):
+    """Configuration for `AlbertModel`.
+
+    The default settings match the configuration of model `albert_xxlarge`.
+    """
+
+    def __init__(self,
+                 vocab_size_or_config_json_file,
+                 embedding_size=128,
+                 hidden_size=4096,
+                 num_hidden_layers=12,
+                 num_hidden_groups=1,
+                 num_attention_heads=64,
+                 intermediate_size=16384,
+                 inner_group_num=1,
+                 down_scale_factor=1,
+                 hidden_act="gelu",
+                 hidden_dropout_prob=0,
+                 attention_probs_dropout_prob=0,
+                 max_position_embeddings=512,
+                 type_vocab_size=2,
+                 initializer_range=0.02,
+                 layer_norm_eps=1e-12, **kwargs):
+        """Constructs AlbertConfig.
+
+        Args:
+            vocab_size: Vocabulary size of `inputs_ids` in `AlbertModel`.
+            embedding_size: size of voc embeddings.
+            hidden_size: Size of the encoder layers and the pooler layer.
+            num_hidden_layers: Number of hidden layers in the Transformer encoder.
+            num_hidden_groups: Number of group for the hidden layers, parameters in
+                the same group are shared.
+            num_attention_heads: Number of attention heads for each attention layer in
+                the Transformer encoder.
+            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
+                layer in the Transformer encoder.
+            inner_group_num: int, number of inner repetition of attention and ffn.
+            down_scale_factor: float, the scale to apply
+            hidden_act: The non-linear activation function (function or string) in the
+                encoder and pooler.
+            hidden_dropout_prob: The dropout probability for all fully connected
+                layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob: The dropout ratio for the attention
+                probabilities.
+            max_position_embeddings: The maximum sequence length that this model might
+                ever be used with. Typically set this to something large just in case
+                (e.g., 512 or 1024 or 2048).
+            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
+                `AlbertModel`.
+            initializer_range: The stdev of the truncated_normal_initializer for
+                initializing all weight matrices.
+        """
+        super(AlbertConfig, self).__init__(**kwargs)
+
+        self.vocab_size = vocab_size_or_config_json_file
+        self.embedding_size = embedding_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_hidden_groups = num_hidden_groups
+        self.num_attention_heads = num_attention_heads
+        self.inner_group_num = inner_group_num
+        self.down_scale_factor = down_scale_factor
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
\ No newline at end of file
diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
new file mode 100644
index 0000000000..04877d41b9
--- /dev/null
+++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
@@ -0,0 +1,44 @@
+
+
+
+from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert
+
+
+
+def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path):
+    # Initialise PyTorch model
+    config = BertConfig.from_json_file(bert_config_file)
+    print("Building PyTorch model from configuration: {}".format(str(config)))
+    model = BertForPreTraining(config)
+
+    # Load weights from tf checkpoint
+    load_tf_weights_in_bert(model, config, tf_checkpoint_path)
+
+    # Save pytorch-model
+    print("Save PyTorch model to {}".format(pytorch_dump_path))
+    torch.save(model.state_dict(), pytorch_dump_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    ## Required parameters
+    parser.add_argument("--tf_checkpoint_path",
+                        default = None,
+                        type = str,
+                        required = True,
+                        help = "Path to the TensorFlow checkpoint path.")
+    parser.add_argument("--albert_config_file",
+                        default = None,
+                        type = str,
+                        required = True,
+                        help = "The config json file corresponding to the pre-trained BERT model. \n"
+                            "This specifies the model architecture.")
+    parser.add_argument("--pytorch_dump_path",
+                        default = None,
+                        type = str,
+                        required = True,
+                        help = "Path to the output PyTorch model.")
+    args = parser.parse_args()
+    convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path,
+                                     args.bert_config_file,
+                                     args.pytorch_dump_path)
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
new file mode 100644
index 0000000000..b006cbe8fd
--- /dev/null
+++ b/transformers/modeling_albert.py
@@ -0,0 +1,331 @@
+
+import os
+import math
+import logging
+import torch
+import torch.nn as nn
+from transformers.configuration_albert import AlbertConfig
+logger = logging.getLogger(__name__)
+
+def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
+    """ Load tf checkpoints in a pytorch model."""
+    try:
+        import re
+        import numpy as np
+        import tensorflow as tf
+    except ImportError:
+        logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
+            "https://www.tensorflow.org/install/ for installation instructions.")
+        raise
+    tf_path = os.path.abspath(tf_checkpoint_path)
+    logger.info("Converting TensorFlow checkpoint from {}".format(tf_path))
+    # Load weights from TF model
+    init_vars = tf.train.list_variables(tf_path)
+    names = []
+    arrays = []
+    for name, shape in init_vars:
+        logger.info("Loading TF weight {} with shape {}".format(name, shape))
+        array = tf.train.load_variable(tf_path, name)
+        names.append(name)
+        arrays.append(array)
+
+    print(model)
+    
+    for name, array in zip(names, arrays):
+        og = name
+        name = name.replace("transformer/group_0/inner_group_0", "transformer")
+        name = name.replace("LayerNorm", "layer_norm")
+        name = name.replace("ffn_1", "ffn")
+        name = name.replace("ffn/intermediate/output", "ffn_output")
+        name = name.replace("attention_1", "attention")   
+        name = name.replace("cls/predictions/transform", "predictions")
+        name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm")        
+        name = name.split('/')
+
+        print(name)
+        pointer = model
+        for m_name in name:
+            if re.fullmatch(r'[A-Za-z]+_\d+', m_name):
+                l = re.split(r'_(\d+)', m_name)
+            else:
+                l = [m_name]
+
+            if l[0] == 'kernel' or l[0] == 'gamma':
+                pointer = getattr(pointer, 'weight')
+            elif l[0] == 'output_bias' or l[0] == 'beta':
+                pointer = getattr(pointer, 'bias')
+            elif l[0] == 'output_weights':
+                pointer = getattr(pointer, 'weight')
+            elif l[0] == 'squad':
+                pointer = getattr(pointer, 'classifier')
+            else:
+                try:
+                    pointer = getattr(pointer, l[0])
+                except AttributeError:
+                    logger.info("Skipping {}".format("/".join(name)))
+                    continue
+            if len(l) >= 2:
+                num = int(l[1])
+                pointer = pointer[num]
+
+        if m_name[-11:] == '_embeddings':
+            pointer = getattr(pointer, 'weight')
+        elif m_name == 'kernel':
+            array = np.transpose(array)
+            print("transposed")
+        try:
+            assert pointer.shape == array.shape
+        except AssertionError as e:
+            e.args += (pointer.shape, array.shape)
+            raise
+        print("Initialize PyTorch weight {} from {}".format(name, og))
+        pointer.data = torch.from_numpy(array)
+
+    return model
+
+
+class AlbertEmbeddings(nn.Module):
+    """
+    Construct the embeddings from word, position and token_type embeddings.
+    """
+    def __init__(self, config):
+        super(AlbertEmbeddings, self).__init__()
+
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size)
+        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size)
+        self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, input_ids, token_type_ids=None, position_ids=None):
+        seq_length = input_ids.size(1)
+        if position_ids is None:
+            position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
+            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros_like(input_ids)
+
+        word_embeddings = self.word_embeddings(input_ids)
+        position_embeddings = self.position_embeddings(position_ids)
+        token_type_embeddings = self.token_type_embeddings(token_type_ids)
+
+        embeddings = word_embeddings + position_embeddings + token_type_embeddings
+        embeddings = self.layer_norm(embeddings)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+    def get_word_embeddings_table(self):
+        return self.word_embeddings
+
+
+class AlbertModel(nn.Module):
+    def __init__(self, config):
+        super(AlbertModel, self).__init__()
+
+        self.config = config
+        self.embeddings = AlbertEmbeddings(config)
+        self.encoder = AlbertEncoder(config)
+        self.pooler = nn.Linear(config.hidden_size, config.hidden_size)
+        self.pooler_activation = nn.Tanh()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros_like(input_ids)
+
+        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+        if head_mask is not None:
+            if head_mask.dim() == 1:
+                head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
+                head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1)
+            elif head_mask.dim() == 2:
+                head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
+            head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility
+        else:
+            head_mask = [None] * self.config.num_hidden_layers
+
+        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        encoder_outputs = self.encoder(embedding_output,
+                                       extended_attention_mask,
+                                       head_mask=head_mask)
+        sequence_output = encoder_outputs[0]
+        print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape)
+        pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0]))
+
+        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+        return outputs
+
+
+class AlbertForMaskedLM(nn.Module):
+    def __init__(self, config):
+        super(AlbertForMaskedLM, self).__init__()
+
+        self.config = config
+        self.bert = AlbertModel(config)
+        self.layer_norm = nn.LayerNorm(config.embedding_size)
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+        self.dense = nn.Linear(config.hidden_size, config.embedding_size)
+        self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size)
+
+    def tie_weights(self):
+        """ Make sure we are sharing the input and output embeddings.
+            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
+        """
+        self._tie_or_clone_weights(self.classifier.word_embeddings,
+                                   self.transformer.embeddings.word_embeddings)
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
+        hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0]
+        hidden_states = self.dense(hidden_states)
+        hidden_states = gelu_new(hidden_states)
+        hidden_states = self.layer_norm(hidden_states)
+
+        logits = self.word_embeddings(hidden_states)
+
+        return logits
+
+
+class AlbertAttention(nn.Module):
+    def __init__(self, config):
+        super(AlbertAttention, self).__init__()
+
+        if config.hidden_size % config.num_attention_heads != 0:
+            raise ValueError(
+                "The hidden size (%d) is not a multiple of the number of attention "
+                "heads (%d)" % (config.hidden_size, config.num_attention_heads))
+        self.output_attentions = config.output_attentions
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(self, input_ids, attention_mask=None, head_mask=None):
+        mixed_query_layer = self.query(input_ids)
+        mixed_key_layer = self.key(input_ids)
+        mixed_value_layer = self.value(input_ids)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+        key_layer = self.transpose_for_scores(mixed_key_layer)
+        value_layer = self.transpose_for_scores(mixed_value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        reshaped_context_layer = context_layer.view(*new_context_layer_shape)
+        w = self.dense.weight.T.view(16, 64, 1024)
+        b = self.dense.bias
+
+        projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
+        projected_context_layer = self.dropout(projected_context_layer)
+        layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer)
+        return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask
+
+
+class AlbertTransformer(nn.Module):
+    def __init__(self, config):
+        super(AlbertTransformer, self).__init__()
+        
+        self.config =config
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.attention = AlbertAttention(config)
+        self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
+        self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        for i in range(self.config.num_hidden_layers):
+            attention_output = self.attention(hidden_states, attention_mask)[0]
+            ffn_output = self.ffn(attention_output)
+            ffn_output = gelu_new(ffn_output)
+            ffn_output = self.ffn_output(ffn_output)
+            hidden_states = self.layer_norm(ffn_output + attention_output)
+
+        return hidden_states
+
+
+def gelu_new(x):
+    """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
+        Also see https://arxiv.org/abs/1606.08415
+    """
+    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+
+
+class AlbertEncoder(nn.Module):
+    def __init__(self, config):
+        super(AlbertEncoder, self).__init__()
+
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
+        self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size)
+        self.transformer = AlbertTransformer(config)
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        hidden_states = self.embedding_hidden_mapping_in(hidden_states)
+        hidden_states = self.transformer(hidden_states, attention_mask, head_mask)
+
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (all_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (all_attentions,)
+        return outputs  # last-layer hidden state, (all hidden states), (all attentions)
+
+# config = AlbertConfig.from_json_file("config.json")
+# # model = AlbertForMaskedLM(config)
+# model = AlbertModel(config)
+
+# model = load_tf_weights_in_albert(model, config, "albert/albert")
+
+# print(model)
+
+# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]])
+# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]])
+# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]])
+
+# # sequence_output, pooled_outputs = model()
+
+# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1]
+
+
+# embeddings_output = 
+# print("pooled output", logits)
+# # print("Pooled output", pooled_outputs)
+
+config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json")
+model = AlbertModel(config)
+model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert")
\ No newline at end of file

From 91ccbae788de11f0f5ffb862421e345b27a20a76 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 29 Oct 2019 21:21:57 +0000
Subject: [PATCH 203/269] Accepts multiple sizes

---
 transformers/modeling_albert.py | 143 ++++++++++++++------------------
 1 file changed, 60 insertions(+), 83 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index b006cbe8fd..f3cebdc3d9 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -5,6 +5,7 @@ import logging
 import torch
 import torch.nn as nn
 from transformers.configuration_albert import AlbertConfig
+from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new
 logger = logging.getLogger(__name__)
 
 def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
@@ -32,14 +33,14 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
     print(model)
     
     for name, array in zip(names, arrays):
+        print(name)
         og = name
         name = name.replace("transformer/group_0/inner_group_0", "transformer")
-        name = name.replace("LayerNorm", "layer_norm")
         name = name.replace("ffn_1", "ffn")
         name = name.replace("ffn/intermediate/output", "ffn_output")
         name = name.replace("attention_1", "attention")   
         name = name.replace("cls/predictions/transform", "predictions")
-        name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm")        
+        name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm")        
         name = name.split('/')
 
         print(name)
@@ -84,44 +85,22 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
     return model
 
 
-class AlbertEmbeddings(nn.Module):
+class AlbertEmbeddings(BertEmbeddings):
     """
     Construct the embeddings from word, position and token_type embeddings.
     """
     def __init__(self, config):
-        super(AlbertEmbeddings, self).__init__()
+        super(AlbertEmbeddings, self).__init__(config)
 
         self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0)
         self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size)
         self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size)
-        self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
-
-    def forward(self, input_ids, token_type_ids=None, position_ids=None):
-        seq_length = input_ids.size(1)
-        if position_ids is None:
-            position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
-            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
-        if token_type_ids is None:
-            token_type_ids = torch.zeros_like(input_ids)
-
-        word_embeddings = self.word_embeddings(input_ids)
-        position_embeddings = self.position_embeddings(position_ids)
-        token_type_embeddings = self.token_type_embeddings(token_type_ids)
-
-        embeddings = word_embeddings + position_embeddings + token_type_embeddings
-        embeddings = self.layer_norm(embeddings)
-        embeddings = self.dropout(embeddings)
-        return embeddings
+        self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps)
 
 
-    def get_word_embeddings_table(self):
-        return self.word_embeddings
-
-
-class AlbertModel(nn.Module):
+class AlbertModel(BertModel):
     def __init__(self, config):
-        super(AlbertModel, self).__init__()
+        super(AlbertModel, self).__init__(config)
 
         self.config = config
         self.embeddings = AlbertEmbeddings(config)
@@ -129,6 +108,7 @@ class AlbertModel(nn.Module):
         self.pooler = nn.Linear(config.hidden_size, config.hidden_size)
         self.pooler_activation = nn.Tanh()
 
+
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
@@ -166,7 +146,7 @@ class AlbertForMaskedLM(nn.Module):
 
         self.config = config
         self.bert = AlbertModel(config)
-        self.layer_norm = nn.LayerNorm(config.embedding_size)
+        self.LayerNorm = nn.LayerNorm(config.embedding_size)
         self.bias = nn.Parameter(torch.zeros(config.vocab_size))
         self.dense = nn.Linear(config.hidden_size, config.embedding_size)
         self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size)
@@ -182,39 +162,47 @@ class AlbertForMaskedLM(nn.Module):
         hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0]
         hidden_states = self.dense(hidden_states)
         hidden_states = gelu_new(hidden_states)
-        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
 
         logits = self.word_embeddings(hidden_states)
 
         return logits
 
 
-class AlbertAttention(nn.Module):
+class AlbertAttention(BertSelfAttention):
     def __init__(self, config):
-        super(AlbertAttention, self).__init__()
-
-        if config.hidden_size % config.num_attention_heads != 0:
-            raise ValueError(
-                "The hidden size (%d) is not a multiple of the number of attention "
-                "heads (%d)" % (config.hidden_size, config.num_attention_heads))
-        self.output_attentions = config.output_attentions
+        super(AlbertAttention, self).__init__(config)
 
         self.num_attention_heads = config.num_attention_heads
-        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
-        self.all_head_size = self.num_attention_heads * self.attention_head_size
-
-        self.query = nn.Linear(config.hidden_size, self.all_head_size)
-        self.key = nn.Linear(config.hidden_size, self.all_head_size)
-        self.value = nn.Linear(config.hidden_size, self.all_head_size)
-
+        self.hidden_size = config.hidden_size 
+        self.attention_head_size = config.hidden_size // config.num_attention_heads
         self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.pruned_heads = set()
 
-    def transpose_for_scores(self, x):
-        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
-        x = x.view(*new_x_shape)
-        return x.permute(0, 2, 1, 3)
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        mask = torch.ones(self.num_attention_heads, self.attention_head_size)
+        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
+        for head in heads:
+            # Compute how many pruned heads are before the head and move the index accordingly
+            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
+            mask[head] = 0
+        mask = mask.view(-1).contiguous().eq(1)
+        index = torch.arange(len(mask))[mask].long()
+
+        # Prune linear layers
+        self.query = prune_linear_layer(self.query, index)
+        self.key = prune_linear_layer(self.key, index)
+        self.value = prune_linear_layer(self.value, index)
+        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.num_attention_heads = self.num_attention_heads - len(heads)
+        self.all_head_size = self.attention_head_size * self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
 
     def forward(self, input_ids, attention_mask=None, head_mask=None):
         mixed_query_layer = self.query(input_ids)
@@ -248,7 +236,8 @@ class AlbertAttention(nn.Module):
         context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
         new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
         reshaped_context_layer = context_layer.view(*new_context_layer_shape)
-        w = self.dense.weight.T.view(16, 64, 1024)
+        print(self.dense.weight.T.shape)
+        w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size)
         b = self.dense.bias
 
         projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
@@ -262,7 +251,7 @@ class AlbertTransformer(nn.Module):
         super(AlbertTransformer, self).__init__()
         
         self.config =config
-        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.attention = AlbertAttention(config)
         self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
@@ -273,18 +262,11 @@ class AlbertTransformer(nn.Module):
             ffn_output = self.ffn(attention_output)
             ffn_output = gelu_new(ffn_output)
             ffn_output = self.ffn_output(ffn_output)
-            hidden_states = self.layer_norm(ffn_output + attention_output)
+            hidden_states = self.LayerNorm(ffn_output + attention_output)
 
         return hidden_states
 
 
-def gelu_new(x):
-    """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
-        Also see https://arxiv.org/abs/1606.08415
-    """
-    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
-
-
 class AlbertEncoder(nn.Module):
     def __init__(self, config):
         super(AlbertEncoder, self).__init__()
@@ -305,27 +287,22 @@ class AlbertEncoder(nn.Module):
             outputs = outputs + (all_attentions,)
         return outputs  # last-layer hidden state, (all hidden states), (all attentions)
 
-# config = AlbertConfig.from_json_file("config.json")
-# # model = AlbertForMaskedLM(config)
-# model = AlbertModel(config)
-
-# model = load_tf_weights_in_albert(model, config, "albert/albert")
-
-# print(model)
-
-# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]])
-# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]])
-# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]])
-
-# # sequence_output, pooled_outputs = model()
-
-# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1]
-
-
-# embeddings_output = 
-# print("pooled output", logits)
-# # print("Pooled output", pooled_outputs)
-
-config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json")
+model_size = "base"
+config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size))
 model = AlbertModel(config)
-model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert")
\ No newline at end of file
+model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size))
+model.eval()
+print(sum(p.numel() for p in model.parameters() if p.requires_grad))
+
+
+input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]]
+input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]]
+segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]]
+
+pt_input_ids = torch.tensor(input_ids)
+pt_input_mask = torch.tensor(input_mask)
+pt_segment_ids = torch.tensor(segment_ids)
+
+pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids}
+pt_output = model(**pt_dict)
+print(pt_output)
\ No newline at end of file

From 139affaa8de89c6ef16a4712d24a22a8fea91eda Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 29 Oct 2019 22:45:24 +0000
Subject: [PATCH 204/269] Albert layer/layer groups

---
 transformers/modeling_albert.py | 80 ++++++++++++++++++++-------------
 1 file changed, 50 insertions(+), 30 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index f3cebdc3d9..440ccf2bce 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -30,17 +30,19 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
         names.append(name)
         arrays.append(array)
 
-    print(model)
+    for name, array in zip(names, arrays):
+        print(name)
     
     for name, array in zip(names, arrays):
         print(name)
         og = name
-        name = name.replace("transformer/group_0/inner_group_0", "transformer")
         name = name.replace("ffn_1", "ffn")
         name = name.replace("ffn/intermediate/output", "ffn_output")
         name = name.replace("attention_1", "attention")   
         name = name.replace("cls/predictions/transform", "predictions")
-        name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm")        
+        name = name.replace("LayerNorm_1", "attention/LayerNorm")    
+        name = name.replace("inner_group_", "albert_layers/") 
+        name = name.replace("group_", "albert_layer_groups/")    
         name = name.split('/')
 
         print(name)
@@ -104,7 +106,7 @@ class AlbertModel(BertModel):
 
         self.config = config
         self.embeddings = AlbertEmbeddings(config)
-        self.encoder = AlbertEncoder(config)
+        self.encoder = AlbertTransformer(config)
         self.pooler = nn.Linear(config.hidden_size, config.hidden_size)
         self.pooler_activation = nn.Tanh()
 
@@ -133,6 +135,7 @@ class AlbertModel(BertModel):
                                        extended_attention_mask,
                                        head_mask=head_mask)
         sequence_output = encoder_outputs[0]
+    
         print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape)
         pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0]))
 
@@ -246,18 +249,18 @@ class AlbertAttention(BertSelfAttention):
         return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask
 
 
-class AlbertTransformer(nn.Module):
+class AlbertLayer(nn.Module):
     def __init__(self, config):
-        super(AlbertTransformer, self).__init__()
+        super(AlbertLayer, self).__init__()
         
-        self.config =config
+        self.config = config
         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.attention = AlbertAttention(config)
         self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        for i in range(self.config.num_hidden_layers):
+        for _ in range(self.config.inner_group_num):
             attention_output = self.attention(hidden_states, attention_mask)[0]
             ffn_output = self.ffn(attention_output)
             ffn_output = gelu_new(ffn_output)
@@ -267,42 +270,59 @@ class AlbertTransformer(nn.Module):
         return hidden_states
 
 
-class AlbertEncoder(nn.Module):
+class AlbertLayerGroup(nn.Module):
     def __init__(self, config):
-        super(AlbertEncoder, self).__init__()
+        super(AlbertLayerGroup, self).__init__()
+        
+        self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)])
 
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        for albert_layer in self.albert_layers:
+            hidden_states = albert_layer(hidden_states, attention_mask, head_mask)
+
+        return hidden_states
+
+
+class AlbertTransformer(nn.Module):
+    def __init__(self, config):
+        super(AlbertTransformer, self).__init__()
+        
+        self.config = config
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size)
-        self.transformer = AlbertTransformer(config)
+        self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)])
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
         hidden_states = self.embedding_hidden_mapping_in(hidden_states)
-        hidden_states = self.transformer(hidden_states, attention_mask, head_mask)
 
-        outputs = (hidden_states,)
-        if self.output_hidden_states:
-            outputs = outputs + (all_hidden_states,)
-        if self.output_attentions:
-            outputs = outputs + (all_attentions,)
-        return outputs  # last-layer hidden state, (all hidden states), (all attentions)
+        for layer_idx in range(self.config.num_hidden_layers):
+            group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups)
+            hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask)  
+        
+        return (hidden_states,)
 
-model_size = "base"
-config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size))
+
+model_size = 'base'
+hidden_groups = 1
+inner_groups = 1
+config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups))
 model = AlbertModel(config)
-model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size))
+
+print(model)
+model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups))
 model.eval()
 print(sum(p.numel() for p in model.parameters() if p.requires_grad))
 
 
-input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]]
-input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]]
-segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]]
+# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]]
+# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]]
+# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]]
 
-pt_input_ids = torch.tensor(input_ids)
-pt_input_mask = torch.tensor(input_mask)
-pt_segment_ids = torch.tensor(segment_ids)
+# pt_input_ids = torch.tensor(input_ids)
+# pt_input_mask = torch.tensor(input_mask)
+# pt_segment_ids = torch.tensor(segment_ids)
 
-pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids}
-pt_output = model(**pt_dict)
-print(pt_output)
\ No newline at end of file
+# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids}
+# pt_output = model(**pt_dict)
+# print(pt_output)
\ No newline at end of file

From 12290c0d5ce8475e884190b6ba480a8b3e671b3e Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 29 Oct 2019 23:19:02 +0000
Subject: [PATCH 205/269] Handles multi layer and multi groups

---
 transformers/modeling_albert.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 440ccf2bce..90e9b162e6 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -136,7 +136,6 @@ class AlbertModel(BertModel):
                                        head_mask=head_mask)
         sequence_output = encoder_outputs[0]
     
-        print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape)
         pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0]))
 
         outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
@@ -260,12 +259,11 @@ class AlbertLayer(nn.Module):
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        for _ in range(self.config.inner_group_num):
-            attention_output = self.attention(hidden_states, attention_mask)[0]
-            ffn_output = self.ffn(attention_output)
-            ffn_output = gelu_new(ffn_output)
-            ffn_output = self.ffn_output(ffn_output)
-            hidden_states = self.LayerNorm(ffn_output + attention_output)
+        attention_output = self.attention(hidden_states, attention_mask)[0]
+        ffn_output = self.ffn(attention_output)
+        ffn_output = gelu_new(ffn_output)
+        ffn_output = self.ffn_output(ffn_output)
+        hidden_states = self.LayerNorm(ffn_output + attention_output)
 
         return hidden_states
 
@@ -303,16 +301,16 @@ class AlbertTransformer(nn.Module):
         return (hidden_states,)
 
 
-model_size = 'base'
-hidden_groups = 1
-inner_groups = 1
-config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups))
-model = AlbertModel(config)
+# model_size = 'base'
+# hidden_groups = 1
+# inner_groups = 2
+# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups))
+# model = AlbertModel(config)
 
-print(model)
-model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups))
-model.eval()
-print(sum(p.numel() for p in model.parameters() if p.requires_grad))
+# # print(model)
+# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups))
+# # model.eval()
+# # print(sum(p.numel() for p in model.parameters() if p.requires_grad))
 
 
 # input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]]

From 1b92564330aa2f40b065a0b9a2a94a28a595bbc6 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 29 Oct 2019 23:23:18 +0000
Subject: [PATCH 206/269] Reorganize and cleanup

---
 transformers/modeling_albert.py | 287 +++++++++++++++-----------------
 1 file changed, 132 insertions(+), 155 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 90e9b162e6..c6662cb6d3 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -100,6 +100,138 @@ class AlbertEmbeddings(BertEmbeddings):
         self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps)
 
 
+class AlbertAttention(BertSelfAttention):
+    def __init__(self, config):
+        super(AlbertAttention, self).__init__(config)
+
+        self.num_attention_heads = config.num_attention_heads
+        self.hidden_size = config.hidden_size 
+        self.attention_head_size = config.hidden_size // config.num_attention_heads
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.pruned_heads = set()
+
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        mask = torch.ones(self.num_attention_heads, self.attention_head_size)
+        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
+        for head in heads:
+            # Compute how many pruned heads are before the head and move the index accordingly
+            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
+            mask[head] = 0
+        mask = mask.view(-1).contiguous().eq(1)
+        index = torch.arange(len(mask))[mask].long()
+
+        # Prune linear layers
+        self.query = prune_linear_layer(self.query, index)
+        self.key = prune_linear_layer(self.key, index)
+        self.value = prune_linear_layer(self.value, index)
+        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.num_attention_heads = self.num_attention_heads - len(heads)
+        self.all_head_size = self.attention_head_size * self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
+
+    def forward(self, input_ids, attention_mask=None, head_mask=None):
+        mixed_query_layer = self.query(input_ids)
+        mixed_key_layer = self.key(input_ids)
+        mixed_value_layer = self.value(input_ids)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+        key_layer = self.transpose_for_scores(mixed_key_layer)
+        value_layer = self.transpose_for_scores(mixed_value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        reshaped_context_layer = context_layer.view(*new_context_layer_shape)
+        
+
+        # Should find a better way to do this
+        w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size)
+        b = self.dense.bias
+
+        projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
+        projected_context_layer = self.dropout(projected_context_layer)
+        layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer)
+        return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask
+
+
+class AlbertLayer(nn.Module):
+    def __init__(self, config):
+        super(AlbertLayer, self).__init__()
+        
+        self.config = config
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.attention = AlbertAttention(config)
+        self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
+        self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        attention_output = self.attention(hidden_states, attention_mask)[0]
+        ffn_output = self.ffn(attention_output)
+        ffn_output = gelu_new(ffn_output)
+        ffn_output = self.ffn_output(ffn_output)
+        hidden_states = self.LayerNorm(ffn_output + attention_output)
+
+        return hidden_states
+
+
+class AlbertLayerGroup(nn.Module):
+    def __init__(self, config):
+        super(AlbertLayerGroup, self).__init__()
+        
+        self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)])
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        for albert_layer in self.albert_layers:
+            hidden_states = albert_layer(hidden_states, attention_mask, head_mask)
+
+        return hidden_states
+
+
+class AlbertTransformer(nn.Module):
+    def __init__(self, config):
+        super(AlbertTransformer, self).__init__()
+        
+        self.config = config
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
+        self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size)
+        self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)])
+
+    def forward(self, hidden_states, attention_mask=None, head_mask=None):
+        hidden_states = self.embedding_hidden_mapping_in(hidden_states)
+
+        for layer_idx in range(self.config.num_hidden_layers):
+            group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups)
+            hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask)  
+        
+        return (hidden_states,)
+
+
 class AlbertModel(BertModel):
     def __init__(self, config):
         super(AlbertModel, self).__init__(config)
@@ -169,158 +301,3 @@ class AlbertForMaskedLM(nn.Module):
         logits = self.word_embeddings(hidden_states)
 
         return logits
-
-
-class AlbertAttention(BertSelfAttention):
-    def __init__(self, config):
-        super(AlbertAttention, self).__init__(config)
-
-        self.num_attention_heads = config.num_attention_heads
-        self.hidden_size = config.hidden_size 
-        self.attention_head_size = config.hidden_size // config.num_attention_heads
-        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.pruned_heads = set()
-
-    def prune_heads(self, heads):
-        if len(heads) == 0:
-            return
-        mask = torch.ones(self.num_attention_heads, self.attention_head_size)
-        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
-        for head in heads:
-            # Compute how many pruned heads are before the head and move the index accordingly
-            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
-            mask[head] = 0
-        mask = mask.view(-1).contiguous().eq(1)
-        index = torch.arange(len(mask))[mask].long()
-
-        # Prune linear layers
-        self.query = prune_linear_layer(self.query, index)
-        self.key = prune_linear_layer(self.key, index)
-        self.value = prune_linear_layer(self.value, index)
-        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
-
-        # Update hyper params and store pruned heads
-        self.num_attention_heads = self.num_attention_heads - len(heads)
-        self.all_head_size = self.attention_head_size * self.num_attention_heads
-        self.pruned_heads = self.pruned_heads.union(heads)
-
-    def forward(self, input_ids, attention_mask=None, head_mask=None):
-        mixed_query_layer = self.query(input_ids)
-        mixed_key_layer = self.key(input_ids)
-        mixed_value_layer = self.value(input_ids)
-
-        query_layer = self.transpose_for_scores(mixed_query_layer)
-        key_layer = self.transpose_for_scores(mixed_key_layer)
-        value_layer = self.transpose_for_scores(mixed_value_layer)
-
-        # Take the dot product between "query" and "key" to get the raw attention scores.
-        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
-        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
-        if attention_mask is not None:
-            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
-            attention_scores = attention_scores + attention_mask
-
-        # Normalize the attention scores to probabilities.
-        attention_probs = nn.Softmax(dim=-1)(attention_scores)
-
-        # This is actually dropping out entire tokens to attend to, which might
-        # seem a bit unusual, but is taken from the original Transformer paper.
-        attention_probs = self.dropout(attention_probs)
-
-        # Mask heads if we want to
-        if head_mask is not None:
-            attention_probs = attention_probs * head_mask
-
-        context_layer = torch.matmul(attention_probs, value_layer)
-
-        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
-        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
-        reshaped_context_layer = context_layer.view(*new_context_layer_shape)
-        print(self.dense.weight.T.shape)
-        w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size)
-        b = self.dense.bias
-
-        projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
-        projected_context_layer = self.dropout(projected_context_layer)
-        layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer)
-        return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask
-
-
-class AlbertLayer(nn.Module):
-    def __init__(self, config):
-        super(AlbertLayer, self).__init__()
-        
-        self.config = config
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.attention = AlbertAttention(config)
-        self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
-        self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
-
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        attention_output = self.attention(hidden_states, attention_mask)[0]
-        ffn_output = self.ffn(attention_output)
-        ffn_output = gelu_new(ffn_output)
-        ffn_output = self.ffn_output(ffn_output)
-        hidden_states = self.LayerNorm(ffn_output + attention_output)
-
-        return hidden_states
-
-
-class AlbertLayerGroup(nn.Module):
-    def __init__(self, config):
-        super(AlbertLayerGroup, self).__init__()
-        
-        self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)])
-
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        for albert_layer in self.albert_layers:
-            hidden_states = albert_layer(hidden_states, attention_mask, head_mask)
-
-        return hidden_states
-
-
-class AlbertTransformer(nn.Module):
-    def __init__(self, config):
-        super(AlbertTransformer, self).__init__()
-        
-        self.config = config
-        self.output_attentions = config.output_attentions
-        self.output_hidden_states = config.output_hidden_states
-        self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size)
-        self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)])
-
-    def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        hidden_states = self.embedding_hidden_mapping_in(hidden_states)
-
-        for layer_idx in range(self.config.num_hidden_layers):
-            group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups)
-            hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask)  
-        
-        return (hidden_states,)
-
-
-# model_size = 'base'
-# hidden_groups = 1
-# inner_groups = 2
-# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups))
-# model = AlbertModel(config)
-
-# # print(model)
-# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups))
-# # model.eval()
-# # print(sum(p.numel() for p in model.parameters() if p.requires_grad))
-
-
-# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]]
-# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]]
-# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]]
-
-# pt_input_ids = torch.tensor(input_ids)
-# pt_input_mask = torch.tensor(input_mask)
-# pt_segment_ids = torch.tensor(segment_ids)
-
-# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids}
-# pt_output = model(**pt_dict)
-# print(pt_output)
\ No newline at end of file

From 67b422662c7002319e54679a73d654ba313702ef Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 29 Oct 2019 23:33:53 +0000
Subject: [PATCH 207/269] Documentation + improved AlbertForMaskedLM

---
 transformers/modeling_albert.py | 85 +++++++++++++++++++++++++++++++--
 1 file changed, 80 insertions(+), 5 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index c6662cb6d3..9b3c51fe25 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -4,8 +4,11 @@ import math
 import logging
 import torch
 import torch.nn as nn
+from torch.nn import CrossEntropyLoss
 from transformers.configuration_albert import AlbertConfig
 from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new
+from .file_utils import add_start_docstrings
+
 logger = logging.getLogger(__name__)
 
 def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
@@ -232,6 +235,70 @@ class AlbertTransformer(nn.Module):
         return (hidden_states,)
 
 
+ALBERT_START_DOCSTRING = r"""    The ALBERT model was proposed in
+    `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_
+    by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents
+    two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT.
+
+    This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and
+    refer to the PyTorch documentation for all matter related to general usage and behavior.
+
+    .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`:
+        https://arxiv.org/abs/1909.11942
+
+    .. _`torch.nn.Module`:
+        https://pytorch.org/docs/stable/nn.html#module
+
+    Parameters:
+        config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. 
+            Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
+"""
+
+ALBERT_INPUTS_DOCSTRING = r"""
+    Inputs:
+        **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of input sequence tokens in the vocabulary.
+            To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows:
+
+            (a) For sequence pairs:
+
+                ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]``
+                
+                ``token_type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1``
+
+            (b) For single sequences:
+
+                ``tokens:         [CLS] the dog is hairy . [SEP]``
+                
+                ``token_type_ids:   0   0   0   0  0     0   0``
+
+            Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on
+            the right rather than the left.
+
+            Indices can be obtained using :class:`transformers.AlbertTokenizer`.
+            See :func:`transformers.PreTrainedTokenizer.encode` and
+            :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
+        **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:
+            Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
+        **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Segment token indices to indicate first and second portions of the inputs.
+            Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1``
+            corresponds to a `sentence B` token
+            (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details).
+        **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of positions of each input sequence tokens in the position embeddings.
+            Selected in the range ``[0, config.max_position_embeddings - 1]``.
+        **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:
+            Mask to nullify selected heads of the self-attention modules.
+            Mask values selected in ``[0, 1]``:
+            ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+"""
+
+@add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.",
+                      BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
 class AlbertModel(BertModel):
     def __init__(self, config):
         super(AlbertModel, self).__init__(config)
@@ -274,6 +341,7 @@ class AlbertModel(BertModel):
         return outputs
 
 
+@add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
 class AlbertForMaskedLM(nn.Module):
     def __init__(self, config):
         super(AlbertForMaskedLM, self).__init__()
@@ -292,12 +360,19 @@ class AlbertForMaskedLM(nn.Module):
         self._tie_or_clone_weights(self.classifier.word_embeddings,
                                    self.transformer.embeddings.word_embeddings)
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
-        hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0]
-        hidden_states = self.dense(hidden_states)
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                masked_lm_labels=None):
+        outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)
+        sequence_outputs = outputs[0]
+        hidden_states = self.dense(sequence_outputs)
         hidden_states = gelu_new(hidden_states)
         hidden_states = self.LayerNorm(hidden_states)
+        prediction_scores = self.word_embeddings(hidden_states)
 
-        logits = self.word_embeddings(hidden_states)
+        outputs = (prediction_scores,) + outputs[2:]  # Add hidden states and attention if they are here
+        if masked_lm_labels is not None:
+            loss_fct = CrossEntropyLoss(ignore_index=-1)
+            masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1))
+            outputs = (masked_lm_loss,) + outputs
 
-        return logits
+        return outputs

From fedac786d401a9da31847b45df842bbee185afe6 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 14:39:16 +0000
Subject: [PATCH 208/269] Tokenization + small fixes

---
 transformers/modeling_albert.py     |   2 +-
 transformers/tokenization_albert.py | 210 ++++++++++++++++++++++++++++
 2 files changed, 211 insertions(+), 1 deletion(-)
 create mode 100644 transformers/tokenization_albert.py

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 9b3c51fe25..a1c0d5f610 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -298,7 +298,7 @@ ALBERT_INPUTS_DOCSTRING = r"""
 """
 
 @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.",
-                      BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+                      ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
 class AlbertModel(BertModel):
     def __init__(self, config):
         super(AlbertModel, self).__init__(config)
diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py
new file mode 100644
index 0000000000..f2e37222f6
--- /dev/null
+++ b/transformers/tokenization_albert.py
@@ -0,0 +1,210 @@
+
+from .tokenization_utils import PreTrainedTokenizer
+import logging
+import unicodedata
+import six
+import os
+from shutil import copyfile
+
+logger = logging.getLogger(__name__)
+
+SPIECE_UNDERLINE = u'▁'
+
+class AlbertTokenizer(PreTrainedTokenizer):
+    """
+        SentencePiece based tokenizer. Peculiarities:
+
+            - requires `SentencePiece <https://github.com/google/sentencepiece>`_
+    """
+    # vocab_files_names = VOCAB_FILES_NAMES
+    # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(self, vocab_file,
+                 do_lower_case=False, remove_space=True, keep_accents=False,
+                 bos_token="[CLS]", eos_token="[SEP]", unk_token="<unk>", sep_token="[SEP]",
+                 pad_token="<pad>", cls_token="[CLS]", mask_token="[MASK]>", **kwargs):
+        super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token,
+                                             unk_token=unk_token, sep_token=sep_token,
+                                             pad_token=pad_token, cls_token=cls_token,
+                                             mask_token=mask_token, **kwargs)
+
+        self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
+        self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
+
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece"
+                           "pip install sentencepiece")
+
+        self.do_lower_case = do_lower_case
+        self.remove_space = remove_space
+        self.keep_accents = keep_accents
+        self.vocab_file = vocab_file
+
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(vocab_file)
+
+    @property
+    def vocab_size(self):
+        return len(self.sp_model)
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["sp_model"] = None
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece"
+                           "pip install sentencepiece")
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(self.vocab_file)
+
+    def preprocess_text(self, inputs):
+        if self.remove_space:
+            outputs = ' '.join(inputs.strip().split())
+        else:
+            outputs = inputs
+        outputs = outputs.replace("``", '"').replace("''", '"')
+
+        if six.PY2 and isinstance(outputs, str):
+            outputs = outputs.decode('utf-8')
+
+        if not self.keep_accents:
+            outputs = unicodedata.normalize('NFKD', outputs)
+            outputs = ''.join([c for c in outputs if not unicodedata.combining(c)])
+        if self.do_lower_case:
+            outputs = outputs.lower()
+
+        return outputs
+
+    def _tokenize(self, text, return_unicode=True, sample=False):
+        """ Tokenize a string.
+            return_unicode is used only for py2
+        """
+        text = self.preprocess_text(text)
+        # note(zhiliny): in some systems, sentencepiece only accepts str for py2
+        if six.PY2 and isinstance(text, unicode):
+            text = text.encode('utf-8')
+
+        if not sample:
+            pieces = self.sp_model.EncodeAsPieces(text)
+        else:
+            pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1)
+        new_pieces = []
+        for piece in pieces:
+            if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit():
+                cur_pieces = self.sp_model.EncodeAsPieces(
+                    piece[:-1].replace(SPIECE_UNDERLINE, ''))
+                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
+                    if len(cur_pieces[0]) == 1:
+                        cur_pieces = cur_pieces[1:]
+                    else:
+                        cur_pieces[0] = cur_pieces[0][1:]
+                cur_pieces.append(piece[-1])
+                new_pieces.extend(cur_pieces)
+            else:
+                new_pieces.append(piece)
+
+        # note(zhiliny): convert back to unicode for py2
+        if six.PY2 and return_unicode:
+            ret_pieces = []
+            for piece in new_pieces:
+                if isinstance(piece, str):
+                    piece = piece.decode('utf-8')
+                ret_pieces.append(piece)
+            new_pieces = ret_pieces
+
+        return new_pieces
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str/unicode) in an id using the vocab. """
+        return self.sp_model.PieceToId(token)
+
+    def _convert_id_to_token(self, index, return_unicode=True):
+        """Converts an index (integer) in a token (string/unicode) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        if six.PY2 and return_unicode and isinstance(token, str):
+            token = token.decode('utf-8')
+        return token
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (strings for sub-words) in a single string."""
+        out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip()
+        return out_string
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        by concatenating and adding special tokens.
+        A RoBERTa sequence has the following format:
+            single sequence: <s> X </s>
+            pair of sequences: <s> A </s></s> B </s>
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+        if token_ids_1 is None:
+            return token_ids_0 + sep + cls
+        return token_ids_0 + sep + token_ids_1 + sep + cls
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
+
+        Args:
+            token_ids_0: list of ids (must not contain special tokens)
+            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
+                for sequence pairs
+            already_has_special_tokens: (default False) Set to True if the token list is already formated with
+                special tokens for the model
+
+        Returns:
+            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            if token_ids_1 is not None:
+                raise ValueError("You should not supply a second sequence if the provided sequence of "
+                                 "ids is already formated with special tokens for the model.")
+            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
+
+        if token_ids_1 is not None:
+            return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1]
+        return ([0] * len(token_ids_0)) + [1, 1]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
+        A BERT sequence pair mask has the following format:
+        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2
+        | first sequence    | second sequence     | CLS segment ID
+        
+        if token_ids_1 is None, only returns the first portion of the mask (0's).
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+        cls_segment_id = [2]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + sep + cls) * [0]
+        return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id
+
+    def save_vocabulary(self, save_directory):
+        """ Save the sentencepiece vocabulary (copy original file) and special tokens file
+            to a directory.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
+            return
+        out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file'])
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+
+        return (out_vocab_file,)

From e3ea5d1d8db36091ab840328cf6691c4f46f2e89 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 15:03:30 +0000
Subject: [PATCH 209/269] Docstrings

---
 transformers/modeling_albert.py | 40 +++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index a1c0d5f610..ad8b979cef 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -300,6 +300,25 @@ ALBERT_INPUTS_DOCSTRING = r"""
 @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.",
                       ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
 class AlbertModel(BertModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
+            Sequence of hidden-states at the output of the last layer of the model.
+        **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)``
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during Bert pretraining. This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+    """
     def __init__(self, config):
         super(AlbertModel, self).__init__(config)
 
@@ -343,6 +362,27 @@ class AlbertModel(BertModel):
 
 @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
 class AlbertForMaskedLM(nn.Module):
+    r"""
+        **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
+            Labels for computing the masked language modeling loss.
+            Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
+            Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels
+            in ``[0, ..., config.vocab_size]``
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Masked language modeling loss.
+        **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+    """
+    
     def __init__(self, config):
         super(AlbertForMaskedLM, self).__init__()
 

From ee20201d339b62948acd05783efc48c4e2b466bb Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 16:19:49 +0000
Subject: [PATCH 210/269] Tokenization tests + fixes + init

---
 transformers/__init__.py                      |   5 ++
 transformers/tests/fixtures/30k-clean.model   | Bin 0 -> 760289 bytes
 .../tests/tokenization_albert_test.py         |  78 ++++++++++++++++++
 transformers/tokenization_albert.py           |  30 +++----
 transformers/tokenization_xlnet.py            |   8 +-
 5 files changed, 102 insertions(+), 19 deletions(-)
 create mode 100644 transformers/tests/fixtures/30k-clean.model
 create mode 100644 transformers/tests/tokenization_albert_test.py

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 5c7b0a6197..152d520e7b 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE
 from .tokenization_xlm import XLMTokenizer
 from .tokenization_roberta import RobertaTokenizer
 from .tokenization_distilbert import DistilBertTokenizer
+from .tokenization_albert import AlbertTokenizer
 from .tokenization_camembert import CamembertTokenizer
 
 # Configurations
@@ -57,6 +58,8 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+from .configuration_albert import AlbertConfig, ALBERT
+from .configuration_albert import AlbertConfig
 from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
 
 # Modeling
@@ -104,6 +107,8 @@ if is_torch_available():
                                 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
+    from .modeling_albert import (AlbertModel, AlbertForMaskedLM)
+
     # Optimization
     from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup,
                                get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup)
diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/30k-clean.model
new file mode 100644
index 0000000000000000000000000000000000000000..c91b8acfa56ccfc80e1cdd854ddcaf9b6c44ab2a
GIT binary patch
literal 760289
zcmZUcd7RhN_s5k&L$Y)#Q6X7UnrY9TB`SONgwHlJpPA3r=l%K2G+9#FLzbju2}KB5
z$`(rYuO&r<5?PW$2?_Z<&pq$Y9goNF^VdDE&t1+v_uTE=bML!s=d$L9=OXbVWgC9&
zwk=b-n5hlvrb-V#_M~AW{`3FvVaK25=~iXi4L^C{u%R9(D{FF~MQ&C`_@MuU_x(?}
z`+vfRl)^ieZTs~Vx0R?J%bLFWK#9=5#?3BKWl)pHO8a&@EhuYQwjJR@bwc9fj$2T+
zLs^q<vY^{Z3(9sXYtA3pN?G<+u@o(miA&Sj4K7+B=k**@ik8esuMe&$QH89uTsD3|
z*^Xt+$RZ+{<q+h2fE=B@y_8pdB#^t-O)jN@ft#mJFHy-rc074TDQaalFY{uMAx|0k
z{Ir=Rifm$6LG4_&J%!aL3)QmMb+bxzp*m8K9acaaGf8=IzlTdnlLcw$^GK<bY6`u1
zGNhFfZII1tA1_hl^5yPxeAGl<K0D-@QX!T3Y+A0I4oxKsa`{QmmLdjJYwin>mS-d*
zyS00z^k;P>E-fc`vLRcP^H)HUNnC#ZY(XhiG$B_vUFh&b<-!54mWr)U<P$PF0d0*K
zEf!?_*RPfSE@*|e%zoV^OiF&|H%bvBq??j@vy^|jL3Y36ozkC~Y(aA0dor6z$eR7$
zEyblJzPiLy>4bbd2dY)s4l4WVWUN|(=K-uSsv~s?xpJ@f90w6;dKn@62YqXhE3UJD
zj*@F85VA^CC)3hx;rlL0LcZK)iT{I!KBO<C6;_$d2L<VRv;RMmO3Bta{(nkH=H-Qd
z0j$VKD#*&!<CiX=Uu~PrRLO*I2)8I}nuyC`g^x<bWD(-AcR*T;6?5{|AO6q!>a1Ko
z<YUL&VKVg*LY8?nk;up&?Ut3ei&Pd+k+-t|7NNeH2nT=R>12jZ^8Ut8O2uXp^&)>m
z()ek)qwS}qoTvw_6`ctXcGdc<bX&H(L?#P?G+753wn4U-ip%vYKPwfSt%H##zWKaF
z7m_sEz%|fTL4=Y%`is)PK_nTM!4Iw|WfzI6JeIuVfvoZ)->m_#^qH)5&aNz_PiJ9v
zY5A8W?qX>CuahCER)h5VnUGh2jJ$lws!}%f33=!iDB6P#{P?zCl_EwXv0B-2zNhQ8
zLq8Y!+C`<X;r|jeR+5!6SXP%3rjoS@d0{v-d8QiV!`BJIMn;zQ|E3fvo8g~RAN4e~
zOUu_Uert-6m)q77wA|xanZL((?hi)0Ej0wKDnS;BuHEnZlFVYR369DFP+V4ynonq#
zvKI7^crmY<*Ll&JQe^akN|d+$p_E%q5w&jJ2ilTE6O4WdxuMP@L99BPP08S|eVTM6
zo{+DOUh4|%A!AM?R1z}{&^86VYgri#6sq&tVpX;5GY2TFS3<@qW%|>Qh~xv=F2AnS
zg6RmN8wW|RK{~R>7lbYIc(y(x`~3}F(!D&ow_W@7r3mRPbxEBJZKX#Og?jBM*@Zqz
zeUffF<5OtU?o75`X4U>w%CeY=qi$R8^#Bz}FdWYH0E12{A$?Z>7+3WPa%{cR&rTRU
zrB#}c{S!+T8l?Sazm(>J3fcS<gfX9rXpikJ8(g}))FlX66GV_16W_?YK`f_SBwvsY
zyKgKdjwZBO4;285P^<Mgw<EMwD^9mMvA6#_C^Ar9Gts9H3Y0DjZuJq00i)9Mw?P{V
zxkNTcHG6ynVzP`>MUolBTiN_q$8k)~JeH6ZpN?c2<n>zU-O6^P*)x$sGMkB{WZ0!(
zmSvELWvSJOk9;a7puBY3ZPS9X)@3b8Rm>zAJOY(kurH|P9c2iRp;ex)PB4viUJh-7
zL`ecvYPa7?2~?$I+X2uz*alKHjG&379_8w_5Ykk!Dl3<4_j@U$Vy2Kx$xm$nEOu3b
zsr(Q~x>bcd+>4O0l&C|g9vSXas`N85c)CxSt*n$??)N0(k^e3B$?LQETG@Mvrz?}G
zv@C1&M@dh5%58^3SXqTcBrSWyp{=?BOr~U?w|#_E7FD?POXyM^^YUen&8`D!!8>0_
zxYV~}5yp-)MuUX8B{Q*vG#%FnF#cA{QCImiG)e*8sF(<1QY!}X#-C8eUv-l1z2a|Z
zE4~^<d|{hEOKKBM#3Cq{+*bx<F;rK++Y=J8?jzS%5;A6-ipk0pfT<D8=94<4AG`w0
z>QEU;CmBPJ_{t}%&T0Vzn{ROqj?#@9LeR3z!(Bn{y9gj`x&}0&_b7|YYGKF?1dXqN
zj{f7jknmM2_Z{$8smLG^39=bE;s}tiQL@=u**qB9ipga2NNv&OK0+#4nUIN#8ne(2
z7s$PDfY`#Ymz=%HN6%=pyx8jRQvQL`nSP#5MCiX4PJ*`1h>+32ylkEdWZcJ5?XCAh
z8b`TQgogd&X#jcwA{_mi1yP<&iwRmq(mF8A*y8DGom<ZD_)jUDDu$w*EZzrT_p+Vn
zqv%Vj7}wF~-cx+6N;h7Odl0~~Wx*GbMejmWIHK$SnFZ-KyRIOJgrm}VJ)w|!;<ReM
zwUnIv@-k>APcxcjWNRBwSL?pu;QpRYB`OPY%aPFGSjC!BmkrI&12VOVv0OuqcTa<u
zgsQSyi%YXUmxzuI=_Sy{R6R`f``G6Ypo=u&rz?C8>0*$KNv|J#4(SL>fa6>ITPieD
zOh;j3kbszA-0a)B%G0R`L+Y*LppElbHl5S{G2}`Q1=`YR^@y8%s+w$dMjrkE8bS7w
zCzca3o)ZXW<vK{@&TgaKPXt31bqcek$2JQ?Ad-p}(=xah08_1Q2F4ys*!YYl;Op5_
zJYCS5CC`M`2F=Uz8iJ+~mDyA(Td%V0lJ@bdRDoONEj~-O=~oei*=pHxlaEBb8sy_?
zp03Jf<MQXT7BeM_7ZS38Af5~gj06Rl`X!jPC>q{Ci|+U%h*e$(;e(a5I@Cup1zETS
z+9VK7RM8KHZQG=jYd(>pXOl;#GB7H}5_umX2%}YUY6lDTk$hJ|R%j|&C@?RM831C1
z7P48{coZb*85z$yfuQy9-qQU<h*HC|RQdB<PX|esP(N2fhx$nuV&1#H!6%Q&)XO1^
z_d<1+F|P9!9$=1R6kM)=@u6ejXLmqad6nAzhdt!uvpLJu%9Qy4VNDW|s$xPqyai(X
z?l1ej>(ke(0(9QuX*Y2kxqZ`8aS=8<2k#AOQ+Jgv`!neT67PF3sBs;nb(2&z$OGv}
zp7G|MD1fn9&^BvU1#LVO<-WlG$w(8GifPc+IS>MC$%2e|3<#aoQK0f&!n9XW=By$V
z){WtrW<B%=5GyB@WLx>bPta}4c5%ZPt1rJwxvc57rHC~}BGxzebiG#Ti5;MghpI%z
z^~Mj&e3U?^ym#9{TZae|xkAEq{KxkJgt^|bv3uj+46)i8Rnnq2pv|eQxX(Eh%H$St
zbNU`f16boCSQ#=1(rVM73y6_}p~*Zar=3p7<d>G;;}Dijy<1`iIRMKhU_tX)BU#BF
za7mtjt!mj=A*;hx?}CSPoybB?4jK<+W!DtZyPa<LQDSueL{&oWngV3SWKs3~ra@Y3
zM2c0q$2js{kK~Hc6eDMqB4H8+{J{@F8s|}FuUZ-KB!KZ&6Uh`Kj1~)D@tIUc+!+7(
zQjerf>L;uNF=nHxBK<e`7_sW2D&Ur~X0E?xB69Ff1g#BXS$U#0RM-Z&JgdRy_XV)L
zGKo<Ps6C*qT0uiVGfVq&5UWHLmW+kHA+0U?$f3tVSmqVwGGv&iQW3fHY#%Wy$D9vg
zHBDt>G#dh}0<x0Wkg*Vwd-EX1OFoMMXjPr3S)NqM{L4L^qC3iV<2}u^!rt^MXscXa
z7ur&NlaIhSSdfV`e1wc!H?4aTAY9Rtqtofd^T0w0pu`M@Prc%k={^NJ%WD8(wfo8q
z|AR6CMbMKTUwAUcIz90%q>UlDY_7<zNcC;MPR&bF%*h`6LKxpVhc(D$`vVwT(PA>?
zRwMf#0A^)Ulh?XInW!qMs*JDjDdLHkRpgxBK1D?zIqe9aBASgi$dsd?tx$G+Y?jJT
z{7(c{e<xh%>6ESpU%Cj|<YMELq$@q5Rzf!M$K^o6OlgsfR0aUn?kq3rl8o7p)_F)%
zsX1g_1CZ6N$a;P}l+`VpBahCv_<t+<N%K3QtcX-3kG!u_z{Fq3=H&0Y{QvO;JxG4L
zA0T9*K=Yw`ZD;y8RV?HRGGmShR6SEtcov}4El6?n2R?!>W#cSP075OH4h#sNd<w#F
z)mOHDZK2Wf^v{IoEYWN}BiA(Fu9Rn1Hzj|y@N_=QmT#{%&`h#4@9P@0N~Y1vxO}C6
z@#FH!90W~a<ua=)A*)_ApDZMUY8lzbr_SbeyZ_|?fKXq1O6(XY)2<kM{dgW3cNpZ8
zsKyoK`BMO_z<8GRP}w;?LON0dP!j<#u?H+GyTl-2Xp|H*WPik&e`hOc<3*($RtaIu
z*0Iae?$WW&|KC@7T?=81)~031Lr|7e9vK||EF|(#Q}zOa#$Bx%`bNCwX?kd_T>dV!
zO;)*lLYE>;a+BZpk<x7SrREc8%Ra@tan`qxCYH)XH19^^O>4lYRi!+zk&v}WB+$8c
z`)#%_DL|o^&wz~B76kqnF}jx#wAeXzGI_aO0V_)_9WB~<GRMEq?Fnhhq-#3zdh{R=
ztDr6e<;t$m##|&7cE5hzJ(j1>GYrI~rXQG9Jf5tq#1uxwA2<TgxS^Nk<8t1S0HN~I
zD@ix(L=Y=4qt=DmlcBAlupbABRHY0%*XL51RMW>21uRWv0+m^t@M&TcTVx<Rqz1^?
zE@ZJb=$@rX#^+O))d68u&I8!};kUNl15vkp?Qj`@Nl*8wl}ehH^emUvS3-sDmdzHT
zx+1H+&F7%z*+8D00uU;Uu1vnW51O{Z{#G-Kpa~<IU=WMT5=AU>Jc)2_eHc0{K2k3)
zJ>tn6gV#lKAx%^^KL^t76(BY5!R-9S7RK%)&3+^t%DsTutsuYr>=SdiP{<;)Q~w3B
z(yJ4xoa`VwloF+F*Y{&PkWd%vBedtw9RNa2hQSNUeF_c?w7L%|Vx6#`yj0<n=hz|^
z<nLnuti(v3vx_z-dRmQm@~P7Fg8uS;ln`RWxIE`tf>sx8Z5p=r9uRAYc(N`T=sfr2
z3@~Gr4H%r<F%!Vba{IMO^P!FPO7!lcS0F9dNGzWX0{Qbz5DM=nZI%(Tyma>!$nIYP
zguPrBT$~MI$N2%!n!~p830bxt$ZFqT+HQa_1`4{4t5w=^O}RBeQ32}6>B#C9ElU6C
zjy#a(S^|UvX@42o8p?{oM4c(ncmvx2S?(!anr*wMr|C3tIiowYm6qj1BO}M01Yi|r
zO&mvL+0#KxA~8%8(;7+L-+z8K|5}+rE)mlWd;5q_P+#33HF=+7CEcIZ;FvleA<z8!
zZiCMv!TLsLjG9Y;OhPsC@TEQj%zWBIue{R#T_wY>f(Uy<a+H)`4QVyTTu_j+ulID7
zo>ncG2yG%`%sBLB|8GIf$`2`R@<4=)jMb9>tW6`a7}J-k?`|`EtVmi{g1bKIv*}sR
z%JUwjQIdK|KW3p3Qu!<)Q^zb;GB;#p7J*rDY-%WO?%M#yePt1T4td{4sA53P$Xg#a
zMqrgH)ytqQ*HFDsA;rQGfLTrTzV$agS`;HnCV?h@{e#a6TUH?JmA0m55t3rpandG_
zBr@(;VdZZis*HJ=+oG%_$co-_{_apFC)U1cHDWf%+_r$B_UWqd^Sz)O6*R_rf7*UP
zR;aTt<fYC2AjocjeBa$CuZ}R{Kh?v>)rA<^cVRh5m?52Btv1K>29mO!*>lF}(Wz|C
zZIb^!1~}9bj1pHI2Wf(@&#KY(`V#=CSf0b|GYA^j9ARhVw_(sGV&-mUBUE^=v%wfq
zb@;D6*T+xkdB|fIKwI8v*MrAj1VG+wLPkd{m`F*7%LrP{vDxZ=q3UuFQ_XZzy4(mA
zszQ?OYX3==v=EWWVUQE6in~FrUKx0aJ?68h=q2+PSdw%^9$rk)a>MMJk!wHkG#XnW
z-z<j?6`xup<eM)6j6JMus`X7*d4!Ebf)V!H)gV+1%gW2&6AVY93O4>eYE>ey@=g3=
zQIip=+vxv{q%Z+8Vl>;a#2iBeHsa2nuGabFg#)0Cmt^3^u{KBiCkFlGos*%%VW^nP
zX;)f)IuOiZURZgyPajFSGpno41F`C|*<(oQ838Cw7A#+w`ec`ck7Nf#6Ker%SQ;Qd
z6(B;17HOZcS9ltU_LSyRp^a&0eF&sR5$hv~c#&PF_ROcJ0a~7Qa>i_ku;)hdm_OsP
z^Bf@KR6A~!9Q2Hj!}(4j&q?9JML@<%&+_vA((4UqE1Ie>XvuAj2!~2)3Tw8MI9B1H
zP}Bu!AWJ{>={U*LIdseSAfcR+38tqzegr_O%miIF_}F^D0<XJn1St&y9PVU_fMb3I
zq=Jzs3(*z-LYTTnlT}jRY^PH8g{0c~_Swz@96Qm9Ew={<dqIL;bMB5lg4uL#+Svou
zMb)&GyLq6-P0O2F1C;u2xHi7{Krk<{0WzpNbT})d<>+4iUmbO2b#G`Y&D!I#WBgx;
zy07#--u{Y5WcpzLS6b)RcZNY*p2?)lI@2OzhUqfgN2K*QaUB62j!kODzYwxbSt~}c
zSal?wLra3{Bvz*=xV1$gTVSYrQ~^sD!BqP|0@CV<McbsaxYDP|N8-t>bgBig?SDL>
z)&n;>KBcJXou1bwIRQIxG>{iOd*bV$ZCs56K{l2&^^M*DOzXN+`#Y5|4sGq&=SH7N
zjFq$O{UEfl$e~h_?QerDn(Y(ivkg?M=Ntel37aTtV+*OmGr-1UB*g@U&7kQ^9>PxH
z7I<S7Dm5%a;4S}&63Hhyxso>*5e1grKyu()Mj-5ZS*L{c8@547jN{ub1qsI$UEaXn
zn&m(!1FV0til8s7x19FXf9Udlvhq8hI+xE@<q_+RYdoZ9CfcC~{q9p$tCjbvZCaL^
z6)V4-*9_7M$!9Ur&1j_KS(aPdHRpe(q7)7{C+|SG)bXp_@Uk<AwKof#M7Rw8a2Fph
z#nKoies2w8!%$Xg+IV8P^xf0vKqVNr+_1I26KJS0bbig^k<LKY`fBDc$n5R_#(7TZ
zFMInmsz3==2!qe?P(9WHU4b?k=2O)s5;-|90d1V6WM1I^)B`42emQigaqdj4!&M%@
zltOk#UJqc5>grM#!6PRCnHJ`<oPMjEE}Q71sU;p;wcK<Ekg*?RiBfkbv~@S-bB>ah
zaV0B7-AbN59mvG*c1D5By5C2N=54n$<N+Y`2fv)t9=Bj^L`KXZ7<O)sell{yGXPe4
zTulfwp0&71wmL6aFrSs@U-8LkT*k+17W!oMdL%mj18C!tJ`>X&$q^p{8JAUwteUJ|
zUebuzOt;fA5Mzg>9F?#5#7A+RKPAgP2eF*diyEd(Nw4;iYS_t9){q}SLQy;A0c-gq
znAHqDiOW)@5eUnwzc%;`botGQQMM7prj%Tk5rfm|lw|(^v=JhiQDd!*MSZsdhayw2
zwx@U5xkS^>1!>+I+BlA>nGIf6w*@lgs+D3#2rDa@)-{BlXT8-KFdS8LIp!o%{nZ7`
z3euIbr1yt5X~o&##-(ddAEB5-dNU985u#aL(cEwtfaMz6Ap>cC9GJCBT8)(j=|0$J
zgPJB}-btQjiP2ZSy5PTbANlJ-pF=d_&dr{Vf*5ByM+dBFe3S&%PL%e>N}owpGM|!E
z$|s3pXpKv|kv>UKOsiqM%Q%l{qp*CMbt8z0R*kbd9L&4fL+Zz(Ej{3NAnQR@WISv#
zBqJAw23hkEVUv9%!(Ib}WL(xh0caA5V>PSF$h0RxO!Cnb*1SnCL8CkL>f>Mbe_=I1
zoQt5XFpf*5&)bkDaXUaI|GPf`+o7yUoou(5kd@@TyD~CQ5u3exl}qk(DASX2S--+Z
zuFOZ&bh_^<5HDsn=94x6s7`04&F=(_4<gHhfBL8~4x<BEwOy-{=&(Jzwb@T)K$PaT
zY1g%aAS=wm)qDD=&g8`w?Uv5=FDJ{Jx)8LXo`s=iq`La+L7>K3+PRuM*TVxrP7f_V
z>jx0_M?H0ZxIZ*fsgda6giI=&8>-&+KguVj)6rwkRRC6<4^W>IAi_<+0C{96lu0~M
z%(G*o_V1qzRvI+eOrhjG&H%DPf{|D`6Efgz5L*B-vQ)BHWdx#|T{YY%R+l{O;*Ujv
zLcue>;=@tn0oGvbHd?2AB36Uy-69`kfWm6o)@D@RN3sPop3C(>#=Ubk(HWukN-(Qm
zJTE&>fHIXuJ7f7|6mgGH&=jRwj=vScB+GgzC%aFAHdTnSgYI|-WXMOpLHo@6_ke`G
z0OJMwS!(ddOu(>R0T}Nxpz9w6wVLZnIWE6E?sH?=m|<8S{FINvYKx^ppQi!na11W8
z;Ta1jgR~UpHYR2B&VY5y^B~4~E>cWM@0Xxa!fYTbUh#ji9WKby1<+RGynL>NwQs--
zuYF^;_koO2hQ>rzcKe_)1J%Eul~x5$OS8rPuPQZ$y|conWvXB$-fbm-Nw5KP<%*wt
z{CZsuRR7}t<_t-?Y=X4*>e)vIZ-Fv#r<3tG3-8E(kSyILBe|`OaX8?oS3S~Xmy#X?
z+J7|GVcU6(HI*%hy6ymG5_3D9^ILeDMJ(>^EupC;UgxWK@qg)Qirlfgr*&g3|0->j
z<g`H20%@8U=V-TD_hsYSf|*e9;3bcX_X9A&JLe!&@!|>}r;t@|r71@MgdJTCKrN1f
zHi6_5x-pVDM}t^VY+)FeDe9ELU@(q>;@Oi4hP{_l0M*!I&Id9^P*;|x_f=YS`q}XG
zfBw_G>@^vHus>+mAD@Lb&2|TkwjLgr2enqiRs6{!lnH}ld>LH_Y2~R{@jk5wF!eh^
zKE4#f__uycp>K|}2zVq^$%yNG@=7MEyqr7{fG(ky&LwvfHujt}mfd6jB^kJWyw4|u
zRd^pg3&8lQC@+`Uk3pJfSm-&giR|M(iMkZg<(>ebEOq10&mnAOv5)K{-JgZF?3le|
z%e(*in^VT}4=t%S$aS9(wo0;`tt#rNRri(lKl^wYuHjJ6F%VP-a@$uPaTfMKZu%O;
zWSq{+h;N~2V*IP8tnpFv>YPxJSJ#4A%@U*7j?!=X`~>8gFP9HCc$%r7iG8C_l8+?S
z#v^}h0y3szYG2}f@-Lr=gVDHbY_V%82IESR1Dua{1+kJt&mnoLjgQ2sQyQE2L+yQ}
zG`ABXtbuRe6Uf+O(G!=)IzUrB<g|ZBA3I1gDAF+x+XpD*ERu<^k^fu~OOIDifxZQ8
z0!PQ`7bw!*-N7`^tc>jKGho-K7QvVLd8C%@G?hL6NS_5Ww{2)2RLlx?4hM00=s1wD
zf4Q^yyc{qH$n-t#{O7hC0$|J))N614P-s(}fF;7{Qz4DlNIY(Po3hiu&^w(l=MN)n
z`MJYZIrBmQ<06wt#NjoW1qnyZ3i&`O`=5D+vp)GW0HMb@KkU<s5XMeicWc*O4oz8V
z{@vwD!enHt^XrwelIb8!)#qO80o~;%WcwQcLV;lWLXn!@>=8ZGVK6*z3W$k_`5>>l
z|M^rPnu1$2>FGXx-n>;_QN+ZP!dqnRgFbnzn0Kd<`#%a)8c}q;dxZj41ilMQS#i1P
z>Hk1MPS@fao&};*_9&k`Pta!VFjE$0kG%k>f<PZGdXca(p=*S%=R;Z>vA8KD>sZ@d
z{|1<G!Tifs{`>bmz(7Yezxf0p)apb6*Rs!{{ou*@rZd_1Tn*}#D#oNgYMn>0GOO<1
zwB92)fT((Q{RzbKroIt5=nv?S;Vg^HKmFfnUBylL3%a!OaW;1HSg4@YfDsTcv*cC)
z^rWB{I?mf+w~|oZlu3e(3L(KvQV!aQpmB|LTKyS@w*=9|dYfghwh&g;@O}ddB(ErH
z1;!IPrnYYfV6|XO#T&3awAF%jq}<uTlTpT-BRfJ;JOjk&a-R_2fk0|2{u8r7mh^=T
z4RS233;ECn5)1%02J}>)1HeN8Omh%4Iv&V($N4z)w?w7<d@?||#H(X)beg~5R3D38
zOZ8tl4Z!+fMK?L_3@EDzD;H^YCS)irHbq2#?_7_#Rc}TXi~unmqwY~kX^@E*fl*^d
zjnN5$CehMilx`*z4V5s`K!0jp17H<0J3ba0g~)_+*>-6kUHzeO^H#v3GorZ-!$ld3
zjzN5R-bc5i)CF}Oz&1vAKj~77T$`1+1}!TF^}McLs}w>d2nW*h3ZIrG6Z1gxu>fHw
zBmv_#Q|0vWfF{OxQm(!l%ETC8XPa;hq$)sChFt3tM>sv;5E`MLub`<D*4A42=sIZ2
zj7g^YdMINM$F({=mwtE>kO>tRd|SO|<OfBKG2I6z3p)16-GJ6S`Gk6ZsLJ0D7OGQ1
zxqoN|fEC!Yx6FRhC&8K#Bz44mMX|6(oTP44GL!-9c(tb`<bY=Z!$wW$j;PNI0LFPD
zmke0=%AT(PSyk~Et(7i~G^cXhw0PD3ALzRLf!BQ&oZz|%@vb+3kS1q(Ti^Dv@#jg&
z=Ecy)fgUw*Mdu?Qp;AZO7rucuj*5YeW81FtY4k!yAloZYl6RJYEs*2ad#KXQ7lHiy
zE0A>!ypQ4xn?0KDUa}Q2J3C{M?n?e`1|0IFn<JUNJ%HuU8sB-COx@8#Tzhl1JgyTE
zqdFt0e6u%U6I#W90p$|!;^{<+Y2n`kpp6%|WDMli3J@zaij^dgiM@RiJSS`A&;dRP
z`lBby%?<@A%@q-rrF4Yr6tjY{jThvuqo5IeG$Pp(2wDjVclLO|5TBlL08=k@FAN1M
zX-!m~xd76#W7zB^Kg4_<c#`2Ja!@sZRfJhMz=ZyM7Q}=#vY3q3GIneYe5zPhZ%@2a
z^Z=ckmZ+`s0M{%L`7WaYtZsN)GpjNWZny-@n4t@#<&`neR#Jw`W$K6f*;pX!7KJ)V
zkApJis)~A|AO~OLlSC7#B(i5$4rCo8R08_j>>B`0Rv3urZ<kC3u(CPkEy#s;LK}b1
z@j$KrS4{J9blWZE_d{FpSlo02H1`3Iq#K+bojU`Bfr~SSQL^7d5HwbGL~>6M49n94
zo>_A|jT2cQdp-wkT;ocgw0`#SB0%Gs2?dARKVJ8#@F6J3ON%|N-bym<Lr)hPa$F2r
z4jrncGt0>A&p}MaYPZ)NZvG1(xUAv0R)hAhnrn3dv3fO-3C8SlNHh73k7ft?fy`S2
zWc6dG##QAF(AIFY{-J+ESw*-y%W2Br3RpopOsFf#7h8eCf;a<bOvfhBtaS-XX6mvj
zbr-u~J21-&Lkr8?E;|5NUS<r5%LT1GWX7yOF5bmMID>IsStW<>3S=ef-rSjCr7f8C
zmpTT++mu8$47mI6LD0%Y)vz+IZx3L?tt@J9mGe7*7$=b!*EpFs&*=mdj_W;T!@f`^
zrCJ=>W#E3$VO@%7tDJw3k3hGH%Yq)zp~{)<;kF8&q$a|37Wu0efN73e%FO4ibzeZn
zF8t{wuHmpln2C+OaG;>gUpPMK*_1krZ5jYtl5GPG_w}IwR(@b6f`<<SK_Ym*d@#`e
zi5jR8cbj8;`WR<TDS700fKbHTKpU+$1zHaVGa=MyCu%tanjCqZV3Y>!uPSZYMMFJZ
zF`$3BbUoG6%w2e_D{btlS3i{<Fbssu8X{6QoS-$O_LX0h<R7e_UtB=Yioq68kV8g5
zhuRpLcqS=iQR-PH%P$u}TRX;!a(n_xh0srWRr-kW2>u$2YN5lt+}++l#-@Nws(Se#
zft|z0iA8KI99i$7N_J-I-Lhm9P$+vnp$O#Au^<$ii^%a;5wtwh%<3umaH7wEJPYb%
z`sY+2<GENX*Uy5o>?&D?(6~=L0%8Kfh%DnChcrE5%46@ZJ@9}zJ{49s=X^RB#3Yl_
z9e4e6&{kn{+2X3aLe}b)46byIxcqOvPZrgM!oUTd4w9oeym|}TDx}T;)w1oo9;kHF
z+i4#HSV7MFn%QvJCqQ;Mq%5QS?tpLPx4>b=*eiUeq}7~_rTW+0yB>t@p)We@vXQXW
zr&8^TZI!lSumDDMBieP7kHRrgQqKL|)2U=Gkad4U(=P0TcWox6ex+M>b=XG1&T0;7
zWfZEJQ?K3uTJ@*DyuBkqD~`R8JMucNrH{=dT_Jz&0&V<9^(?DecG)9L-&a<*_mS~g
zWQTl!r%BvXCLHK#Tsl+IsT|tc#*H6(zxRRuU{-*;)5)2}VIVd<;Ag>>)Kz2Z;h@GV
z#&zee^Z1cIVj&T$&SX>Bss_1ppwE!gU}pgu>Jd%?l8iv#pXM{uGo(1HtPjrsGTuT5
zw?M|94Q6`5T7xzpIvfDmvAFqFgES*6J!yLZ;ZQp;AM5x&QlYSqu$5&FxDbF2QR&=U
zURBB(g%h^^GCb;|<K9TC-Nr^Y%#zzoMeUPw6*A4@8mW9=?XzGLn2-b0(D23v<W~(E
zKRS1-m_N+{S@rS>?a_3L9Re^Dbiz4Rx}ANGI#6THttu|6hqeZC(`8oPyu|0l_Q%%I
zZN~UmdXb8DxJHpsx0SzkW1&ljsrZCochQPII1U)Os9$_Sve!eobuZ67OduR;MkJ{X
zdG}2osnhEU_%~1T2u3>H%#5B4!Xn5i&8t%&Ol0b%diZomYm{8I4u)qaU_4d~=vOYs
z-3N^fv(@s}41!j=ZdEV|-#6R-tcpm(;{<JV>)A`X&4IFPF`;ln{O%_K!m6v@w|^Si
z^3ofybfQi3L40%A1a|rYFk?2#p~9#|kh%?3;eGWEgtaKnG{`NGpB1u9dAdVA)sydo
zgo9{xo+(oe%Kbh7w32Z8h~;@uMD|+jlVXQZbv^wf5ah094gJKY;BsItqUWJaz67%x
zVU|cp#VVfzRwZ=_8Tvg4q8lBNBi0ZMyK0dgBwVyu2V`7u?VHsd*a7Q(9=hP8?G|kS
zfeQ_vw#lc!=uzi9Y6kxXW*Ji`Yw5V``@7GCYrc7oM5q7Z5snU3CI0vmBy4tkb66lM
zVBF{y+#N4%_qUJ4+a)OG@+NIc))<@@BAMi9voBt-EqH0FQtxM7K(19VQ~~zG)l$Aa
zfVECMp&lZ#Z<$A!V{|`t@{S&H*RccHucgn1mw=GbTMAe<L6RkCUe0L+V!XSJmRfPP
z><(sfQSTOQ(2rVsNRI?_#JaVehn%Q@Htq#v&8h}usow|Msxh*tUM>Ch^$}3*26<WO
zum#u)>lOYZ_xDi9kklXGv(X#gTtHL6%E9c1yKy&YtCl`9K($`!4x&v@Ax|Gd*b33T
zn(E3!y@0I4p}%GQpe(z9@mn4{9NMy@uTp<$IS|C8RE?#@Iq~*V%u;c;F)rT^f<|Mb
zS&_jO<i6O_(|t+?O1sZE?<}9PDhvBZo(FAWVu#7-dbR@Ab?dNvyfPe;oEbEpxPYLI
zv8;34#S|uwmld@d#yI=X1=BGXd5AR#+sV8{Vjg0JfjY!xr%E6O1=q$0)IeB;SV{7r
zkkZCQO*SemM?;p1rjObh{*6mPjrSPcE-zEZ0FY(9T1Z|U3t^1uV6Ox8wc~)OOe!hQ
zY0%PU*tDeN;OhWP@OpiQ?fH#2f*21RzsMaoLt1|7O0AkX<rbet4Qed@{+teCjf)5I
zQ};tzAuLLe_H8o(Pz`OfrVskWY@0N@`bR*lJ8ExdiA0V)p8~Yz=XDa^%3*c1_8BlM
zrBAv1I2X$JtYU*zGY^tVVud*CIf4|2OQg)3Z(+t@DJ&oy3XaO><<^A&RyP&_m_agf
z%SS+D6GUYBGJ>TsCaz|>23h<GSlA8Jkay~5p4O38K3@S%(Yh3@{)(^_N5v^_+*$xD
zF3lAht@}yqfx>FyM1=oVT%P&~%*s&bc=_>XpAVKob`xL!;Q`#V7=|AC3m_b0^fimP
zJfn!^!P8=l0)In?TH!9o2GX`k+Y-b<6T6>4_HPPQ5-C1{+d)zp9V4F6u>DobT#?%e
z((=cmg$GAVXv;s9jOKNBRoBWxmH6XoZ3naivf8@0LUi1HdQUKXNJdKiUW6zvg{Tr8
zd}3C-?ym6t9ev_FHV)_K@lY2qV?!k>OZWH5Ftxf(Ui=^*g^4gDYm~NDQ75KAUhd`r
z)|Lzf($8nYDwPZ4Ol1Qa<7hKUkHZ1N<wIY2O({56kEik@EKFW9e;{FNmwLTvnURHx
zm;|sk<r{Pf^K)Y?{07v;{Nv+&EW5qW3iZho!HjPlF=^s2hXPQjKGJs9sf4Ydh#S;p
z#4sOSmudLYodpnT02_b3J2Oe4uoC^H<JmqBy?u`@?EZ5+f}60idck=hHXt#<mfZBp
zE(A5!*z(jhNVkgttd^y{pgdL$7M5R(Ijd~z8X(HIt8#5d`Uos@oMZl)1|T{kXD}^B
z6Qs7v*J+m#w(|6XKac)h4q)Zsxs-9E@z-N4EiNhhjP-vr1>)<2DyFXlGtJ@fr6Aub
zZS0|4MLq7l{Td*OR8Qb#w-Po1Gg7Fr;-}j{;4Ph%4!2uGJs&&g4#JkNt3qUoPl!gU
zP4(C5AXXA%o%6hU?s1^7A^ORKbD*?b)}g(gB5VTZp(vFamwn{qxu8~GCV5tg44BIm
zvw|6v;}Uz;a?u09zn=HGIF}Dw20izJ&xMJtpa&sx>MLN>ihCZj->_h$T5eiG&;+hy
zHg!<I#KALO`Z~tMr69(VT?fQ_eey?O#ynF8$E@&)O?sI}bxuq=pDn3e4r)apC-nUv
z1wxg^nFi#7FF>q@j5+<~t5wj*NnLDr{D!azHN|PY4uPE;AvNq}nCxCyODtm&ewetJ
zo7O1iTDc(2))Nky;4q^g{eSk^vb%P2?)nQ565{^msT#CqOUGo}4N%4&9_$4ft~9f<
z+mfBR5yFN~%$4%mCLgzAK;Lqi`v<fY9CvPls}-=?+h$LKtsqtyjt2+->nU7<m$Yc-
zioy~Sm$5rRcvV&1u4w5ITo%<W;|T$w(t3NiS4YAoIyD+<=UChc$k-|1$q~q@`!wc2
z-N@nmeL)Zdhl~XchEj9Z+gV*az#DHIUOb_IDMD!{z(V8bgFubZfU764zOn~M*gmR`
z>MwdhPcRCK6y^6`1g#+YJs11;ILzmuu^Afr9SdS|=CZTIj`t~W=;k2k_Y(n(lakW|
zIxyuFP%BqoLgVZ^CMORAGkG&7T&q-xGgW3(ey5AUoDc(<9U*MxCK?hlvI^3kGhywT
z&yHd)!l}MNwy*Z_QtT-1P}11Z(^$OT*o(`O6sWZboBvn@&ynU?AlrJ=QB4(z`EU+6
zlt>Pr3|Xc?m{gZIKu?bHar6<SKyGhnjDz!xnx3~G17v()wbEPDbwqq_JSY{^#~{XB
zN!YZT1#?<nyvnC=5A(T!Ri<Cp7)uwUEhl)IF56dLyaC$8lw+H4(5;XZpew2WY1meQ
zrI#ZDx%oE0(BgnUF`G^&#XggOjhSj1L)zaC9cq$W%E)X*RBLMG-n)EcXSe)aX_J|{
zD6ma@z^CTorXClrSHz@f_N%-cGt)=n)iizTruU;hAAN!;kVTIHP-b03wwX)ND#Pj(
zmpNS!ZSyQ3wbl0kFMYw{Vh-H$B0=M)-W}xa_cDO#T0X&{nKt67idl_Wf+ytlSD{Ui
z+DXb6HU6y=@PV)SEZmY>7XTMGV)PzXBP>?pZxPRwJTXcZyaQ#N<RW^WkounobR`<d
zg^PXs%82w>0%f&~yE7M^hw7IC8u#iG<F48qy9~^@Pw8!(CM%$wVFYt}b0w@)#%WI5
z|5U(o;N}}+61{QlmtfYy+|5u4to;ha@=WOI?X6!!hiw+lDKD)CqCvWLmAAels9Pl6
z*?;&wMA$mcMMJwlehr}IhpP6IxtpNFF-v!IW{UkmQ7eTTH*Podn?jV5%5vL)Af-6h
z>?gNEm~4BO%MVTWC}qzCT5mL|DtgP@ZTYJtF|~NpSnoIU7{dowX=shFntP1irkms{
zY25<M^nlf_ENcmAmB5silbO4DI+NhAWTVnB$Qp*b3A;m}k6PXvZ3u@l#}tm^0lt3i
zz|5Hir#hRx_3&|BCrhVO%R~DRHhI8PJc%3ZMf=*{szASMSZg^z?%dDEWMSh*rUhN?
zU#_tJ)!l+@EspO&(DXrXEoJ2RasX?4?$~LqA5+BIMwbwYyt{6-M^8|zgSv;hs|9!T
z1~Y-M$_(Vo!=Wv6HHP!n<B<ReK|K%e9Z1*;au(Z!{G*6vi2;kvG<|J{L0~qvqp$wb
zT_A%_@W}&ATa5M#6*1}1inw8<x%%fJJGzWZlT(0<Cwe<WVY@~^mvRj<Qvp-h#3**d
zZmOGkh6nxXN_w8<^WxNsCC|FEJ;0k&9JX&bAHXW9k2KKsk>Ma=&(UY|m{;a1WV*}g
z^FJ5(2x{w5QJoqGGQHygIJr?t6Ej{OfefpJwnEgLg(h5^Y($vq7y>t>d=@&x?UwTq
zIIz(kd0!sHGS$N)c{_kMIpH)0<3ASwObygRge#A>&!bm>ny|tb!Dxx{u|6##MGUEN
z0AZ!w5d}AV8X=zIbvIs~zLHqzFN59h8Z=dK??CBT*Wp@^={ZGOU8jyw%&MMZsA9Nz
zRS~O)9%snW*ZCN{Ql-nWt&Io^Rb2wCn?M|^tFv#)0XG6zXVfe6v1AUj8jFywH-TGi
z$s3!b&S(QC0$PcTWj%3S_PKGyOZIo(^4|!3WWnvw#<aVp!&9u2J)*W-My!YL@M)4k
zG$Z%k>0`L_+d!VV7bF}Eih1|U-SzhYSv~YA)j;l4z{IFaqP%)aG`-(PitF=Na_>w4
zV<(=)fR~V?AM{9|PB^>-qlh(rViaSIT_N~*mXGHiH;v0?Mf^C<lG5$%UVa$V_`>^#
zA!Weh046dXA}dJE6Q0&Biu8UG+UjOo+EFq=A(Vq&UH1$@V_5eZXyKo8L2Tus*O8LE
zMxiZp+_S)8zoB>TKOfTAVAsPrX4jWJz~yxM&6ouMx<sNwv|2<sR4qMDkIScugk414
zsSkYJ)0Mgh>G%e8s7676)&HZn09<xR<{iS;$$Jivjqm!TF}(uU<O83SN91UwvzG!`
z%i)M-4|To$5tvmc$}Vi5WspduHYw{rB^b&pyq|vgav)<r&|PFfuKwKOGwF9<LC_EB
z?)aF_)aOf3;~z5z6+ZbZ0IM*@Nygl-eFSE5*naE3fZBUTuOWt&nof*La?NV;I-du<
zqqqF=laIkRM13CK`vt^0l{Plk5|($JHUoxj#g&FY?r#J*0W9w(?Y9uaB;?*{P`$cq
zoAyp5JREv*Q-YQ&j@=yX=))QJDrk++t3u9i24zx<a}P>K=U27|GX85YROhPY{uUr6
z1AU4|UMTa4>{ud@UONNnyxU8TXho0~*Ow4S?dD^$UUN5GH@60|TJg*cqgAw>2h>s?
zNL>d2s}+x?Ch*7(<X6RvDLMxx2>GQWNU85f-5t-{_xCxe#VU{`3Rs`8MI5~r5nX>U
zu*pU5s%6|{e)b_gKX<-B(>|_H*pg@{!rQe6h%v9%>uRN-fN`OZr|Z&qK)Hw1w@lqb
z->mQ%v53>zX{92Tksj6sQta!aV5!t9PdEm|8VU!B>yGvRGlZ*Cz&|H?B$IWXxfh=X
z!eWcXNFc4wfS?)Fd^29dRvL<f*KV(~0E}(zD{9^Ab~X@Z(fwtY5rj%w<($;w^7=&{
zE3HA~$rzaR#dIVli6o>+D^SZ*m26VPie<NqL8W&MfECO8cJ2wMrbQrY3%eGh33sda
z@kVp@qxR2NE(bCl#ACQYre6VVrA09<a)a&gn|z|si$liV41_@3%Ix%8{r~DStL03%
z4ajP$ugbb{bkHO)6Fa`XE0wZ3X7LX?^A1Skf}UI<Q>Q|QYMG(ua%A@ah-IlZE(Qd)
zwTf9L0lS_~Gkl^13sa0>O=g0ap!D>L>rhVulxmA*zvolXVSAyjddBk6LLl0g4ZFl&
zwXoV)hb#)iC{^b-eKNP8rkM|X3n*N|gl{E)PEpjjR_9>nzcT9`FsqwgNtQYP^RZG9
zuEFY!?Pc$S8E5P|-PZBK4}jX0?TSxfByM-L%oHaJ^q+77=qA&b`s{)t3ap<)nDB+i
z^f_4fxWsFUSt`!mZdeJ$NK=nf(JF$zHF`-zX%nOFR|4txl?S+s#13ST0+v;(C<lBE
zWi?EUim8L!DT<gJ>XLc3eVoj_@C_J(*6q%<-xH?rnxq`LmY@~Jtq8`kL5+a3F3FGU
ziDB&FXb{K^KSNvL>9pMSr%z9BL33Je0kFXe3mgZ`YOIaR9)ANHllX+FFX&$X_>`)0
zfsEVgfv8(+22J-Yl^o47alE9oRf~MN^{wZR&B2U+#)O{Tq{&VI)RV=jEZK#y70M&H
zY8}0ESCCRa<@GOjxc<p*9;9G)2|C!+@9xv0N-?Z)t6GDYj<_uWwOT4b2oi-I+|GhE
zS*h$qkp7D1bd{}p8=lL`r~48#c336r%WwJ49!bQVH~;W1AmM0)xhTCqqzRv+bl$c-
z(EqKkzPIiMS(+MEz13G_se&dFJ$|p19~uE<)T^8PS?-e|dOcpHJs$1_7<MT&3Nq7D
z!asdLjh|?Clr-%NX)@q&-8lek(+|k?U>;xU@AF{Wi0g=Zo9~PPptc6r_fTvpNiR9(
z2=G$dl@H(+MkC<79AvQqei))Q=-;zPf|(R~giu{V4>=0NG?F`!6$AbMTtLfF&R55R
z&`R8OEI*N8IEK}7GCJ7Db<S;hxqb+U@x%nj!n@PSmKbx{h*SK39HXO6T~4+Cf}+IE
zA!rR1Q^%JV&Vx2-mhAeh1uwq<6wPq9iasMCta9PY6&$K5X4$A`xU9Pfx}<aF->@YC
zguK-=`L{|7s}JVFBw?!>&(6wGHISB*yVqY(+Hz6@ryjIUX@u;GL?E|hiDlKdj~q13
z`D8WDvo0ljDP#?xwj7F{SOlQ~V3{ttR;f^BSO9cufHX!JKY3gC68|?gd)e+%NXsp%
z8_{aHqY;Vg1I~FFIfgi;j$g$n!^CmI6=23GgENtS8S68uw0HgPALpY4xYhGA>6HM+
zSryj#TdsjLk?8KD3Uwbe0SMu$W5(qgE~#fl?`|^uMgXfxPH(m*Zt^sJuU_`M724Qz
zFQ)1~v*m4IRz0(5O2zFUmaRHyG{`gsLJ1@|F66b^$sVcX0uz1u`Kch5wV4c2>ocZ#
z48yazZeOUFv6!vJ-mq>aG^)>hy<(OHE0gljV+4(bRJO<}>vIK&hPVG+kNf|ax)8(B
zPXbt$Y-rf6D(wbXzQXg2|1YiI3i@)c{ikEoKN_}l6$9`)_-P)rRUtI$1ai%@VAcZe
zu>+FLeGbTY(-DSE?Q1?C7DRlA=?!QTmwulJLG1NrV-l9@897CPQbVvX)U&*Ki+v<q
zL{YPqp8?pY6cnRDf)l}vE{>mC<rA}Qz&L`=KL54H;Ju1xh5{L~+GDzrcAoRQd<$mG
zV^?7n9bfAKj^gMXr~cvr#^OHR<g*O`3<=x>u981~gRqe-nv@g&fU;_^`!-j|BmV@W
zrP%ZSqCpd$8mrSx{FiU>5Mw220=Z5hYX<eJm2<aR1}t^AZL(Ks_F#(3Fr%`8Keaia
ziPv87keDJ?DSTV|ca!Qe4+NaX<mAGZ0Mvm3;<Blg&%jhwGP{F><1-)DQp!3ZR~zIb
zC9M{9xTxuqMVGhsIdIUX`;6Nau}W~A2`1b6L|iiG8d5~gYUiUQIS!EX_VhGP>;vR4
zrNgFSDb8MRF955kdhoCq|Fkm*vaL(XYy0`sjB!<RQ5Wd2Db(1m9%?rq0A@MnottWT
zcb^ESdtTN##MAW=98w1M`cDGgP}TMUpg^8KS=x`FaZ_*iiau&Y*iz__w@ML{g4>uJ
z)*m|D`<E`{aVO+3Q2M93haGjePr*RKMF*vg<s|Na)zbe+4`2si`FG_p0LDi|R}XQi
zKFK32=L)n>w=+G0-l#X!&<jAU5i!0ntnp>Ti-D}snW(mpL0Q3-*%-6JU2%ZY=+M8v
zJXiy5b>Rj_7H>_~335U$sEJ)~#o-!;J8=puRC{N=WEfE>EGsE9^H5d=*3ShBTcwC)
zV<um!bxQzd9AWMx(l28GEMI#9F_5w=fDom!GG;vCP;JrjXRd;@LfmcpyqtZtN7y)O
zPdHZ*V+)nq_8KTF5VI_;|LOGrCIa_ufc%ZnwwO~Rs(rL<k%Gn*kJ&Kw#bxiCd`3)L
zblE!_fjC1`L{7ii=N!|^hbt#RQxE+d&`-Df4Bfj(DEIIwK051PK07lF+BjuiV*z>W
zJpfksnruQQC~0lQ$ft`kHR^t<D233uWYl!R#x>6>xf7Rd?*|GOfr(5eXh?C>hkeNj
zGr;Ksd=_x&Lj+AAyphK%tB*okv3liZ;cQ52MZ4#tt@5O$=JAm?G-zBivi9vJmpu(&
zOz457cH_Cv`ZzWC<plEAD*#r6KDvF(8<1f)XE&tJE6NjZ`!ss5fG<V;55St9=|OwZ
zo$rEJNxb)?HcOeI5Y@^T<@5ImTD9DB_|Gl2zu55}u+)OQFxh!IL1UkNSAl{0GXUcy
zSKVOe3&TGLv!Yq@a4ahWzXma-3G7);G^EKlfL3C@h!wb1NS7RW&sfX<G?iLbTdwmN
zH15{+`^9Hat5)2Pl{PWyMt$B!|924K3AFE(Hn!B=gQd~d%^uM`EIbY0;t?+BsD<h2
ztss`Edf?99rbDT|`1`oMj6B^8%!I;6^vL($<^XWca!qb*;s0d)tj{Vn0!&o>y2*oO
z#4tHjau6rG?FelmRo^MMIoWP!u+olD?+L4(T;9q@4BT3hb-~`df`ubgk|r`Qfm0Q=
zEZuz>O?`7)FqqUAh)!!y7(L)L>h8S=8aGw?tYA$CXsaA9`Bd(dPCf$eBYYHQA0Hu?
zWD1aOoqdFW$E0P9(%Oz}CEIl&94Z1gpv_fjxq_h%X+PEDp}GUWLX}Tc$+-tYTHE26
zmSyB#)QIrq0o@u->rNcXVW5<gdJrz{@7*Tc-IJcu6WA)J57SAex6hWN6Uvs!eL%wI
z=C#ga`$1am`5pltr@I{vV7#jF*u7vf??{g^9wvBBA}+@b^a=IFOV;dz0IUZ3q!Bz$
zKNZ9VUfj$kp95vRvl^cZ&12O0K93-qM|wvO_i=RS)>iB=!pCuwhRmq+KuuAP&bwE6
zK%Zm_WO+3J<sy7CXj0?<4x9_+-6;TLBu;JAe9<olWYTPOWWnMyvN5SX*Dv?v0cZhi
z1Ts&<Ca5$^yriClbq~NO*-ac0D?)_@VdKz~t34Zl!k(mwWXmN!4z5ZuUHLwHxsMYt
zu_R=(0w`}RDa*$YG$GrWq3l1_XTU;1&!psJg{%<giy}j=0x(vIl5&1lL$3y7vB(lC
zBgJbV;M*Q*d-^(`o>v=KSB$vc1C_~2mM6Iz0Zh$U)%BHICqf%r9Aj|dLLxVVSds2#
zj}FE^C>D+btli|Q$<QVrr{HMBB~yGxI;qmS3ls@8TA$?DeVUJ<-jk%MQp7Ymir4v%
z_cW4`Dp`6TB=vN6b3RtW_~ZykO+Po@4`Q;?(};ktmRNi>7|Q6G5Z0S{f01|i^lWU|
zgMh|1t9?!s@;vGX<mJ&%-k9an=@%T?I#QFP9|oiu>D$X5A!vN?DNi-<l+OVP+l&L+
zT50n<fMv<iY*NxnTLdn=Fd_b=fa)Ss_N@8-&m6ZEvU1}~9>CJ5?eQ{zwFg@#bp{CJ
z^jCZ&PL5z}_yPdyusD$4w+PBts<$+G4ceGyp{^&`i{J7H<GQNe+_yofW1lW^@O%DG
zOp@wmKj{OHIFk@FmLeuf*7qE%2XfS6AS;#c5~StqMp~aFlHf!Br`jxerDCa1&Y6K;
zG?RvpfMB3!g-p?4sDiw=$B7i5oB9~c>cD}nZt3?~210&%-y!`eVJlnD>*TgZI$mUE
zOE34SosA`s!e=1XDlxtBB<(&2uo~4Saadp&obm;b@quC3SvcgU6+Rt`Nf*0$C4eb$
zJX<ACukvY}ky8h<_TTt4am*17(p>@5PGpaxf-3Z3-+E9#M50!{f*kk*pfxNEUGpQ9
z?sfG7@^9;Wd~B}lj@4W;`6ocD7QGigllOk{0KR}IbKi{smLqr7vPvO!-d(<0gP|h3
z1{?G{K&Z%i_Hsuf&2dm56`T1#><juLW<mD;)2FD^b=_%yL6dt8E%Fb+k{YW~oBlXM
zF<TLEhe&UM>pVATE2w3wPwBAisA<-*)HLc{q^FagHTMvvH`WroIkN+hrNb`9T}e5j
z%tzv|-FatE+8L-MA6@ed6c4cKWzDlo8vv6Jmfi_%{olnr*8$|*y#RdQ>LcwtK~qI8
z>`vL2pj9V)2v+YO%~Z6cM7RvAOWeZF9-~j|gT;JT{17l>zlt*oT6;}z5YrNUY{6XP
zcI^Xb<ukyiWaqv<E8FFwUI+H`SS9B(&VBf<L%~eFxiRMU<ToD<WcjoHi^QGxaBu`5
zD&}k%OOJvuX*u61=|0f@b>G;2TnX*5y`{r3J|Zq&7<XnXV8z)d^%y!83e_oxva~xE
z+5}yL-B>Bjte5P0oKNel-AvE-o#@lDL#vl|LwpqXushq@4-{fJzzij;hY~hf>LDtt
z^ixjuNUrF<da_Ot>$iCO$?(%5O&Dx0N0~`~je^E9M*|JA<C#7eUFjm~A;Uqep6<(r
zj74K2K2c6@JzrMo6V)W@wOf?c_!!&)E69#Yn~LD_ir*m{{+}~o#vYp_`7`U2)ad}%
ztq5&3Y%nAKF?BwZNG$Z;d`B^BKYe^MkRvV!2y3a<31<n8j`4Z9eLzMwU*Yo#5AGVI
z!`Q}HFwKy>KoOcOTPMekCuoAgf{nlC)jl%bdTIe2sffj3w5^q?*LoUrV1q=jgND`c
z1yn8v-T*-Eyj*zKZ3IhWN0v|e(1lh`viO`4>FHn~LvIJLme2=*be3D8kd>L^2^i@-
z)zj`lo%g0en-Hm{K6<p~UJ&cT9NU_^b0F_b_wnf1Iuv$!5X3kNUk?amn}@(kHN-yp
z%Pb#<ywpXC_fj4Pvj(BoSQeg}4PecjMv0G|<MY5?la<?_^nY{YAWzNxZ*tW7fO-D^
zbYzsQSJGsl--Feu`;2FS=oUE+Lw0|TFnt@Djh5vvLKrLRo*c+#1;S3lTaa|4aq~e!
z<;G<~y%Fjb0$C+dRj!@93LTDl`ns(RHlHo>vDh9I<c7CA%>q0vFZ>VM%2pe7T7q{0
z!j@pw!*byJMx-jm>tHEq_8xJl38SN&rQt9GcS`-J0^ul&yCc><Hcan>nQSql$o@+`
zS>R;wuaA5d>SM|`vOWf|s&O4ECHF4(5nvMa*`k0ksdw&>#_`{Sn1Go6+&gJwe+CL0
z6cg5a8z60v>)S^j`4x&P=z^xxCc@SgxuiCl<vL@<GzDoPwjp$hKOw9<3{Sd{sNDh*
zvZ!y(_V^1rRKrvQ{va`2ApiE5`k1)+X2MpVj#YmznXGik72m^a+No5Id%`x53$_I@
z@nQ(T6!f<O#;ki!TKDfF+xv9cjJmH#popa_)T@c&9|cT6`oNUd;qaC~RMA;r58v7U
z&wAV$X(#UP^Wc~pXRQ}o0~oVhB)y}pPod74nqy@<AeAazewRH6+kQ|xo%6B=9<Q(;
zL2Pf_s=deD=?{03Wkw^WpNr6|a&l!y;)T6HzktN3|3oJbajM{|^}yahFri;d?YS@E
z5)*o(%!~;Ig)O7d4~r9R4{<}rxdct@3}mft?{G@nu09e%a-Jgj?mm(lkXZOt9O4ny
zQPtdmI7JcD#5(kCWH}@Z>f6c1{)Da9vXW!q!~3ZqLk9RP_|~&NAG7RGkLUo3Aln=c
zVj74cpk9_A0Uh=L_hoC&1&#zVC5RW*^Y<DBEDlc`U`A(tI0^`6)a&?`V+f-VOk)xG
z>sScOKdoOh*yT9rQXg>NHQM`l5X()M@~Qw=C}h=9=OMfv^*&PHL7>J5#wr^3uAx39
z>r7`B_~9fVlaU&b#-0jkYGRH_?7u!%%s7Z*R%GNDa+=S_o}>vR(+Dxps(#6JrxVMH
z4|2H7%F_y%9;4NXGkmr+`j{K`u`@x87hP!Xr=&@OKHF1fo$YBxUL3p5g*IMwhILom
zdY%VnW$I@%xSu&3fHL`BDGf1ygiqpb2dO{uDHnq=&DIp<1`S$G)x5%@Xigl&^5Z=_
znChSKIfP$f3FL<=Fk=)e4QC|PK91U<+<QD-lOA*1clEygtOm?7w>8?BEP$1dLzK=N
zyX8Qvt<gVsW9Tl0%nYuMRqW&GUhJO!R2+>`<LO&@!eP#lDq6lp0AMWY;!dtp+J-YO
zRq9n{Cf7F=wBptCk8b{HV>F$uu?eD9BS#Spf#s5N%OwQ0OEG(VH`ZW|9fNTMt$6m&
zjDl^(1DJlYSJn&NTK{_$waVe>LqEhIa;1kNQT_T1pU0a3WL%k#ft;y`=@HLG?R_)U
z|M#+DayV80DJcjKf2d3N;kSU9j@97JvFELjmbtCc<#|P@9p`)B-bT=>5pf5TH%<bu
z46&&&Vj%Vblfg`q^eMfmz{I<13Sijb`68XJzqcr4Y2D{N)kEm_d%&y!lSrp~0m8a+
z3_#D`(un9qJCI@b5ht8BS>@WN)9zNVRG@Am1L^vJkB7ws%dCDF{)idAfNExRI^7eB
zP+wk}+V3HPw5C37dqx=6dnMm%xYQB*4UoCBpiTZ5a5gJtMPsUPO}6}D59!xj^wRF#
zvwa@)5gzb;%;$k$#NChkf1TQYtu&l)rYu*@A#C}xRn!HRjDN~QJUof|;^y)UP^j_A
zQ54v0o(J^9&vCg@0n3L^eshyDE|)I=3OlZT$#36Rp-r^NFk;_bIz&NBr~QpqkQWuQ
zw!}prvm|*neI3k%!m{<oH=&FjeJnSSnQ!?ddN||O(6it6k)la`+2+jm8Ud=#4MGKs
zL2e%4g75*fF{mp%hCE8%cQK$TjdKa+W27H~c#-KHCV6owP{>eHXTT*2lp2-e<0Z>{
zF77KefgJV;h}BYW=W!iBE<3LBse)>D{O2fb9ow0e^d2iuPbz90@zDh?v`O%d&x^SM
zyCL24JH@Or?1r^Ej0y2?0j+=Pxk2rENaHfl+k0y_`o9Y-_s`!1X%%&!3MIvy-#x^b
zrdA|5`wyV7?z*CpwwnP;b?;vxPj2-QI5lA;Jmy~y(8XB!RyN(cR3kMW&{p%Bfs|Gd
z*s$DlX>$1Xp!BAyqzv1^M<_jpf;O*i0cvW(Yd>%71ZAA@Z3=uJo3#Wm&e))(B&W1R
z&?!8S$9M4&Fj1xChFyJxN<1m}R7Yz7IH)elLJgMGv`X(Su#HyCnkHRzM>C&yY((^)
zXGS{i3u02``T>tZ@F0~!mW;y!eX??4XAoQBHC_S2?6E&^NwocX$}**`J$VOGIv)UO
zS+jw58<#l;`V@S3hmQqxg|?!iygq``eR;4?L=V(QUnJQbDAdh3$G|$oOgaRNK|?=i
z+PVkfP`BK36l!#wSpjIgsaKAibd;xg!K%O9FwoQ5X=T=N(8dXFfZFk6$Ag5k2ZAa;
z(I?=9BqddYp;^f3*Qw;_Ax7xwr7Tv$+EEX;3i4+ofK3wM-l4>Rd$!kd@yQnE7(xC$
zjj$;U#=2k6fHL{jC!%yDuHX&>GDUQs^T7EOoj>wS(6ArTeuK|}wDR=@d0Yq12QXPM
z(&gp0;hxqjbaM9yXcHh_XIKJsiLw4-KodyN5Z7rUAMw$c>G8<sa+G|oU>J>sH9W3r
zM7TVnv)I5Waf~7K8d;_lutaJ*#7H!)5(J5GnM>wsIBbd_uOARtq=;3Jxl^u7LZbFP
zdrGmEplK$3ITM7Ze$&#xCdfijeo>0}xRnJNf|R4JFfvQn<ZCYJSd4N&rfTkdTpeU5
zkF?aaMcJo7u%z%@Hyu+CZJe?bVpAypG+5+FR4%^6|4A>>!Mj$Guwf$l$b7R)0Zj0$
zSutVjrmpQ6K;wbol$8PNqW2ZE+mLW>FCjT=R@W=|!|I^#bgB&`KMsg0>TYg=hK*qa
zmy_48gbwS+kuc^74g;?OGZ7{-IDls}F6SMu0k*ou@e;w7euW~X+V|@%S6%C4(A~MT
zB5fytSbf|k(=F47-vDT}#xokz%M%J%Yeu6nxo9G!@x+|3cUt%W>CIqPl)g<S1C+Lm
zx!}cHS*mFBZv`|CD^u=uw$774tSGL7FqB`ZfaQ(cP!MLv@9qGzKIE)fc0*`~yL{3r
zZrTL$paLdY%;{BXzFMsidc@<RtM4IbW%H#WeF1En`#`*O`pS_?TYCM5sy?0g*8M<M
zaSRP|;DeAf6z?d@w1<7_>LRk<;SnEId+e~sAT2Yu4?uedKH;IjJr9M8fI?PV_e`Z4
zDP}+EF=r^${`lcj9xL4tl6fzHg(HhwF*zk@_adkXlr=2F>iGEp#t55)sD3@cK}e;d
zi`?-F|5`KYj*1bT(ms0?&;-M}rd9^N1x>YiDo*~@u!*slaTcX(-}8u$m9YjXTMR<}
z(V|RSLeTOry~u)x=_eixidDGf>e^+Ig2uRes#)Io!UHTL*-=hk31BsGPePFBnJ<Bi
z8zic;$c10|RQlmjj@rKluz>{^{WDiX!5U6a-)JyYJ*MVrdHNdwYYd&d^~gc?{tnFe
zN7dO#=`b`-L5rxXemv=Uk$4T5aZ<2%5S~!Pl5(b!#E7p9H-a44*(GUtPeJ1(U#!AW
zal#KiGkbzcdj8^(m>zZTG2=#rw`=s_g;O>Vhh|D*`MYc*VQa#AwZzr@3T-Uo+Kwdz
z%R#5#eH=b~oRD=&n|u+aI|18qGZ0y(IMaAe!<Hp}B$cu_{7=6DKj3elG(0epO$u4p
zQTzG7N}7Ba4%9UB>Q)e%f*UTA|0QUm;yONkS*Nq5+w4=?Nmt{t!S02pJyC<tnt+?M
z;`t(r_m#~7d_V3d9k=%p^f-=B^(as(n^VS={LsQ9_K2QbQwCz}R&o#Gf|iiRQR(F*
z+&No$P+uIbayF<(b^|Px-k0}IcZW79>cfNDpRQ^RWGWcrMveFgI@hXcdyzun<d=}`
z+Cy26*=i0xTkYj(os8vHrLE)ZL90Hn$Yx}^qE@b1C%8$`0f^QMxCzmz6Je79ivV?9
z&hG8AVYXvA?`|<2v5$|ZQb%QPQ^e%WYMYVooX!9yDjvR*{C<!n0rc)8pDJylL&aI3
z;&?CZy8zPIeY?n^2l@DXDkv?@yF$Yy3jx_j!$fPyO8LPS?Acf5XwW#&3+{PYp+Kn3
zJVb`q6L+S1fEgQfRNe~f1??)Il*0xP4jY#z@lHGf(nMu0&|G3T5{R{-n{Rp_?f+*b
zTF??(47C59j8aOJYTU1ne6O_S=T^WtY9D+okO{%vq;xmM4?GUg+JHkM^{I?00+Y4O
z7*jQDwNzj426=RlkM6F2=Vkl}An1u6Z}EDSQkFm`Ku#=&0F*j8OJWsEL@_OaN}M~C
zFn1>pmJ?4RWCGDu?CMh>ja$rvw3@D>7M})awZb~XxmR4yIvvPLR5yk|ZdSnJu+4Bb
zqSqSZa3zDc+Bcj<9OEh*b6@v~%UkDwS^1PEr=1Jw1=>r_9S&{fnA7?b3K)Z>H(D6u
zF7P1F@-x8dba%oCKpo)gWa5PcE!UdJ=+Sa_1R7B=$H`I+TZVcdYzFX7Q4hLTmo-b7
z83VLNi`qDQQ5A?)+I{_%F4?vg$hsug`_(e04hR#|pcKd<hxEx1HYqTx7qn10HtRFO
z`;CLixXe_@O5-8nB3khBND$*Re9nRqWu=0~0?!KSZ6Ensp^^-FglmJ+#u8Q##(bx6
zN9RE;f4!84*Wwe3ATS<PD+>sE{fRQ!VT37ySxa%>U%tB(k}7ctRrb1!u$4Gk_3@_5
zp^a1hb}GguG<?JuK;yKYSzL$1v&RBiG5R!v`~1#a#jM56Aop@(B=xOxwzbT7BH6kZ
zPe(WGX?|ZQ{NL+mY;$z3eer6aCx`O5Q;xe1fU2^=c<OqB#ui?&Y4y?hdjgQvRQH5m
zP4ubx)=qs~rcMShwiz$&R3$ML%tVmW1@}ayO$53t!*N3SxXb6nB9bd1n;QXy#Oyrq
zZl5)8194N0R4wiWqqBCEN)4H8T)|SOfR$oj9aJf-x)0D=*7-sNvd>Hqn^hQ=k5kH6
z#m}6NIn06%B@lkD=YdCn!fuE8G$Ma0V0k(R)j;-s%;!?8A6JPhZ87++rq*_?A|^)N
z&y1M^88$Z0%Hii6$keBOCRn5zWYqJ}#tKifxGD3C1wIO!{|Z^5G)<R{$qfrFIGRhg
zult0wqWh@e-|qs2I-F-kt2%h<2OiT8+_5#t$Oj*SG2nBOnUO9ZL0Bz#3RrLc4gDCz
zSW-Vb74Y$&`&_UFab<u`*Y*oA6EW?&N+~Pbxp?a~xqOAsi2W(5mXMvk1fmbsF%HEv
zY#H?GDf5)FHe!V5^KYx5O_tcR<*L<?#uWF!ot@{P?>uBzX?W%Jdm!zioT#6>hHyA|
z*h|l}P|F_xt+se->wBD$wIE0rHR-XQpcSc;^AaVkNPL!9It6mh1|aKhdK;aOw%XA0
zqT-?Cxpc~rD-W4%1hb;_dk-LwD`FYy;!LH#wh@ZNv(Aut$L~H}6}=y(x7iG0qQO5S
zA=m%y^Wl;=4=u;#?WX&dqOdTmlD&6;wyGp~E@?uU|C_f32gu|dJ&k)d-y>8y6iYVE
z2*yVQd<^FoGqELf*uKT^t)?$pfms7`OZ2N<p~4=-!V-(&Za#^#g{va<R1DQ~55G@V
zg4V)<Blqv_Q|pk!%gL<)OjzL(O**v!qN&OU$k7@!k+_!yy0-PP(Q1akyj;-^h}=2Q
zk!$uKOtI?NF2}d`ae1JG$kj4pPaxx6zj6~u%e??7Dw31M9V|FHDP<ik$fLe{??=$s
z)5%xsc6b*cQ#g$1?#r5i^xq%Q7>(kyDm6+ov<;Up4j=@7+zgez<%B7&R~NaeH^Hz!
z(loewOzI0_MHMsJBQx@?Le{)RoJx1<4{7Y{2PasY915W9vsF2aU}?a#w`6gQJ=*^7
z*-y4R-luKAYlQofg8<-*!wYG3g8x$&{5+E}7@(xoDK+5q9s*+AagyU6=DB$&kS-{R
zAb%)ftnwilNuT7S>-tic3MZTlWO+uj?!!GZ6fvQpuKi@wX+Du22kN4_^%+2>dtB0+
ztCTUpOskIK^4Bn+P^I0o`O@S}AIS`dIHsKqWP-yfo3XFcIRG}z>g{oT;Z9k6`+Q(y
zC`tL!YlKh7YL;{hF9a|tGx@QR)9K*mivW!)_5ah^uG7Up#+5te4x~-gXT{bKQ_g`3
zSSoH`QI|lbDHNts=XGwuRDmF3+=WKUze>>B+>9Pl?PJ>7P99JsT%K{gQ168xo#r*5
zR-yVV9~rnv>9F%UkE3q20IHFF<f#-v6LmvT#%1geY$JPQEr@M#pOF^q*+s@_(D--P
z;ox?19>{b6BcTi`Kw5Qp^Flvs*tQPD##MDBz<6vY%Ci*@E6zrbz4RJI!k)@}iJwos
z2iPg_-pk-o05(!#8@Jb_ij7$K(Yiq1YapJ}^Q?5ajIfo1Qu5f7RF1WO)wSza4V$pg
zc)TdaLt6<YGp<ZiEab}_9yHtpU|FI?>iO6((I<2JBDJL5dovhf9?(S&yoI1wAr^i5
zgwFK}hQqFX5j&6<{)6Gr;HHGfZnJ#qveJGMLF1OCT1u8R(zQwYcgHFGPrY>}Nom-)
z#(2x=#hR%A)>0{2>a{!l{|pB-`SN=j0ll&(zbRm??cO(n=N9(@8PB1wZ6GtIgN1Yb
zp)zm=lqzgmsx)X~!d9wJ8vdh5C<oVZ^*g=s2LX+*TBgnGm9*@5lj7-FP@(eDJ#y0P
zVF0SAE)Q>N*s}BIfYR|%pT06Cb&o?C4{CM9qI>lm5UWDCr)iK!6bt35{z&CdL0hRg
zJfsWqj{;Vbe;ZLwe+JOx6IyYk$6O$706txM_B_JoBH&*XVd2I|F-!kJM*X_T%IAC@
zjBffii0u8m{hKXH@<snIdt-Ikm&kmeP#N7>+~!6gQkP`~{PIi0uqyNEJeG8_RgqB1
zb<@^!k&oeQl7XE28i-Y;G>^<w$mA5!Z?37udf*#AT3rIWHZH;^ya~i;Rf%;%&QrpQ
zu4kV>iy!<Bh!xFh6C=?7d<-3x>5Wa_1+j8i*y^Q(Dk=LA%qoe~4MwxROMEhXWiwn-
z9=g;=;ge)oFHiXBKT%vaJ^B+6Yc#BPT;b5HM}O+0MN(0<vn^4?<^{fD%s1|=k=lI*
zY@9Pu^_HiVHZ^4*CQqz@G>N$<I9LPjyb8#;=3JNqjRzD!8*n3+LEjR#?C=2Q%K}Q9
zn)7wWx7R>fZthAEUG4ZEfDmj$f)}}eQ-YlID|1(`C5-m)#h$-3XvuH^;b5*odj1Gx
z<z&<!>O`eOZBXkc4?wT;iQK-6L43AC)&^u^4mV$~_ZbDqMb7*gI_&0s<b;hrg|h}`
z+5gll=BkU0PQ#%>VZ~R;@4E@e#KSoTGsEt`djR_rBUH!D0LFVP%eig+U(kp^jgnIF
zkB^;GOO9+-pmY@FkBiv7^lRGL@u8=!FK<h@Bq8pup~OoRGCmklP_Cnz`8Zf@(&|b$
ztvL`f?#<Y}9YL!%2X8lQ?-RS{#bllWRsgmm+$G}Dt_6@)L!WSy2})ZOwuS00D*eiU
z$iHWQ8NVYzV+Wsl7OY<>VA0*{?XYw1?m*!XNe{nKNo!M1m-KDwmaRd;&Z=+AsJuTC
zFk^!^%~+r8;p4D~;$8+zt;HR|{*S7&4zRPh+BlFxaqU493M5zvkd238En3_up547S
zyPH+^?k3sbS{#B?tdy4E?!lb^!GpWIgy8c1o_XFom+!AT=ehHaoH=7>&YW?=<(|fP
zMkK}J>mODk=wh)YW89p)GJt!NJZ9^J6+2sjg%mf&cY~m4FVq2}<Al{ATn+Y>SpD?G
z>1zNXAQhze>mh`*GypueHUN2Z(6YojwNK^78-|S%l>2g_X6!!{(CeR)-o@29Mkx*w
znoZT%74Z7Gw#%mrcY#=DVV+Woesw@*WHOIJ={^iZ4Kp^j9G?H=N|>xLCkvtcj3#l`
z2*TzZXX5e6s7#J$#64&Xr1Ro*ii1+Nr(=O!1C*M{K{yJ-^#IKU22;2&VSNBM8h6^e
zW7!RJgt82sV#Z%HkZzB}Mw<c@O~|;^E7DbxEh6>q+8iXcO5M=-UPK`}Jk!o(bmJD)
zzlV;Av$rw9+H<X#CBkK41RNEeJN%v=8UGPoNG|eJj?Z??Kr1d#>SL!}0LWU-`Qpyq
zAlzETI$EVW{~w?(P<on>#oM0#2>)MAvG+a@&cjwdJpx>)R^6ulUUVV(>6uknz0Q6a
z<Dx(3GGONmGn-IO#cYwZ^O#j*#5jVM;Yv%aQlBXduoo;rIeP|i#`q5eSh%n$EZ#(8
zP<f~f?gpQz^DjpKR|UCdJf|Tqx-;4`(#XqR%$}dOgSa+Ly|@ZHd4FgmjD=rp*+ba;
zDo<{1X<)kozfSE1rQj?|4xXGp=DtxWz7(yGQE*(}mp|t4QiekM_<Nz%E+E}~CGMC4
z<QgKW)MWGlIS#uYY|lF5&>GaB(VmpnJeXK1FBJNR<RrNEM6IqApg&p8?R;2Ht}tPb
z-46#cV=z;Zfnf8IIR%}L_r<uQoPs<pwJ7Ws#zm&=xVTt&bS()T6Y=c#KSAJ2&y+av
zr2IizjLU)A3}G}fo_=YXGfswbrMVYD0X{kfpePcWTl(3|sUYsI=?Zt6IvzL;(EP)r
zLPaC!U(U!-2~}Mi8?_?@xl!fSGj0^^+}d;#DK<J2fWlyT#9fYiMfvX%PZ3t)(6az6
zs+<_cN#{U%o|7XxmPn}7dl-@T{1<<?&@38SSjzlxE{MyADRCMNH#<K^(xoTrdh!K1
z66;c?9BhdNIY%8M4!RJM_LFJc?h3=#7FmTF?xRtb{x@M4y27|ripvEMuRJB*QqX;b
zU9VgWvFN4m4E!Isr-xDT&@>1azL#4JSQ%|{DTvv|JtCa4iFReIm}GZ$IS>VFr3Bk4
zOlryf+zMAfxB}eFjo~6)y2dt~B*iXQIx_1Ho_-VIe~~t(*Y>LbBpgY(bYVd;2DbOc
zMOX7*AsOY#8iB7DVtLduFFoY<lQ36pWGr%xlOq9EyltY21n_hLA6^jRiZ-+$0Lx#S
zY37R7cwe-?Xzt~@c_l8p4y2gqxTA^)r`7oNpcYXE)bZR9r*(S-niCoa*0=%EbwzPX
zQo9Op;+>PxBb4I(T9ip?hcx&`qPTpFT9xP)?S_y;YYods+zjOUj*Zq^A)F;U{@CX>
zNJ#=OIp0n&PsPl7JaMnl<I(BhYDWf$&+l+pZlBM%hp<z^kS4x=5Ym$W`v?7a+V&y-
zAbWJhJ#mwWqOtLD)DZ_gnt@8oWL8w4J`UhIvj@<*=i)OnlAibE1$H5ld0g@B^AzEt
zn`7Ln3Lhefzgfw~#m^E}muQJIUm%#5B@Cc=r;klu0(5pL0H<ItdHBm2>0rr;rQ4FP
zfw-ovY_<RPI;0uZ)!f3P3Ak!+M!C7g+c}zS1=o2uN7Km0P%O)oa(pD*1ulAAG=Bn=
zg`;@PW|F{SWYTJ5?oV?hv>ipH`2<ouwhFBZnzgLyh7O6_1X%VNi7U9!=!>O3%ZVgK
zL$cS|QlN7h-;3eg)n7rogsr&V5=#ev4J0<U#w!XswqAybZD;4$S{CV<y6@%y6--4<
z1~T?Lf6q(LYyL>k#gcn~DeQ&fMq%zV$vz$fMY_j<^9-uy#QlYGtk@6Q<<eoa%JrQf
zn!3VEJ8RBQ*n30GJ>5Oh8urCIf}I~Dh<q#Ka(*E&Gra2QoN?!OAfMV&@sb+I@`RCF
ztJc*P;d8G-%Du7t;!tKhM>{MxM=t?Dp`^%}e<{MQSur81PsP%pRT;2F{BeSoTX0DR
zqW`afz>k&ws<vz$&e8t63_!}DB{o>rVa`FKyFcMVWa`^HV&MUq#^9nHo3E5<p5?>$
z)XLCqC!QwLiQEB$atzeWt+Csx&@Lzn7;f9+!9x(4bA3E!Dr`QcC$M<mx)fKf259bZ
zSA53mnHt|4Zx4ZTS88vW9Cxe@iCB#o9nY*w&}Fw}!LUrT;p~Y$hiAHO?1&L@&<JQV
zMYny?A&&%bJJYWyDn(guRE{(uJ%)I#07v3@SRTc3#wW}<V|t2BgJ@UE2}#hd9#i{}
zuAFQ%E*uL+MST`9T~s00jPSgS(i3m32WIxk@sUo~&-+V;Sal~oX6O1qu1Yh8QXTPD
z4UnVHN*u5OG2Bp`3-ii-{98zoTRLN;jCX99V<p!?%$`E3k~5TR6en#20K;3`W6g~T
zYFyT!bo5_yL@w1L2t)n`P&5<@$elNZbau&#R|@jSgt<`oywTkJy%2K}e|hCtZ8K<B
zU(QVR%-6WhYaiJM-KkrERAtC&R30b)JD>}W(Hg3>wFS656;?3YiY%lBEmZ8d6*NLA
zKmFa?5-y~=4Fy3@EVCVmQ{(D5vhw3D0PdB#E{Gw7+W6_+ax`6vY>oT&04SutMeknV
z&|z;NCr(nj)^X&1AVsCM+L|renN?5{lwzrIIYNauM50S{p~EViQfQ@fNIh`Tq8!n}
zPt2(sGS*d(mn@k)eAft8tsd$FE>n~<($X=Zm+uBQWrVpxj<s(RLTlrOq$7T9hH$-f
zUxjPSu~#dQ`OBrwShy{d<I($V-wx?6%?N(Z{!p$ly5gQ#pc~pHVg<=t;W1u_V_>$2
z_1mNzL*A2E2wXTBq$<C>3a4fK@qJ)s3{F`w4OZE(J~;r`rR4&5OFQ?CZa*+X8t65o
zZgmh46-+lxPd^yKZIDg|;&K6Q@bL^%#TIM$A)xdIG$8vb=u8TUh_?>|a<y<A!VWh|
zNBj;+$6w=*1VQekh<;V!LNr(!A{sj$1yaZl)|^r^eo=!sALUtYJe~a0X;fg`oeKL4
z-3pLY?spuJTSjC58plJzT3rNIQ@5W0ZANmXl~YJIms_0#=H_a}_1-B^PBXdJWym|Y
z1{L1*;)1EfGRsOhp5;crfI_a3^ms;eZGFhyQ0rcgJrx`lV3-%{oR$;8yAvOMd1g+e
zM~}WWoek~MN`ahOZnbm33NFb(6?OjfLLjq}b*OsTpo>9Vr3v-%;eVluE+m^T#t6y4
zwwL73xcICQX_09l9$a`}f-RK>yR`&5L)PNyiOC%WQ3)9nj=YR8lEmGTrr1D)xzFW=
zI7ei`S@aK=LzzV=U53SDqDh6NdAy@AvW4HtH?J@(H<6>SBJ7&!`VK|B;c5`G#6GU*
z9fPj}%LxyUmv4l2DTXnMuXhuq<hCx3Qqbw?%!HZY*PB6HyN)tXYLUf_w*a|qbx6)j
z*tY>tP;8Rok=q@{y=crgov_(OFJL=^kj3Hxo%u)%&^M})4ZVHw>K**+w7LF??s?ri
zYXC;>nC0IE@VmxjL31yNi|9A~@SlAjP%$3ZGaGh0_h+nL_u=Fs^F?9iOu9C<>jOC{
zRzR?zFMfPDLmj<oNg7jU<gZu=wa54;bCgEjq+}}@z0Uwyq~Ic4BSTablb+3AV*Y}e
z>?~-Pv$H|&a*)mUgt<&i&3O3~jR0vG9<47zm_Nx#B(8c1K)h#`eOzI)2K`uxCEA%U
zgE)PLZRw^;G3He;w@UTyK_!k7=ybbyPO3Ae)c|%SH1cAv5knF<GWL6epi9^#k4N9X
z4NV!ek~s5S!fp(l3BbhWctubl2I(%Ij=I))AJ7@<;h&y3Ux16Mkq7hdO2h{l(~+{C
z0*-ZrIW_sw!mgA$?fem7Q6e25?EEpbQ)f<LGWethpjhEp<&jT*_XXu%>{DoWMa-0r
z`V5N7Vq*7F5p3#>{l5Tp*4$n1X-?O!&JbA58vB^Vzk+tlOVyl)5(;pPU`K0Ci<z&X
z%_k3BmDolQnfA5CTi@0GO{&dGEdG6l*cGJ3U%MbrLo&nC@ddot;HQjY!OBi2`G|h%
z7f?&lh}dTCRZ~tpta;u%1dFLmZxA~6{QWY@m?kqcv3|@SLCqs{Td@F?xseX87&hBt
zSAnjUBwnNMzy*QGIh{n0UL=21ueY9A#g$^GMKdTZgUqYD;{}112YL!_y@`0u;$Rl=
zPQN_!wh)gd+Hia+GS!l2ZMRtp!Yr4eX51%Q<J_3o@lW|L9XrOzrJ*e^xJy89u!Ddi
zcYHV|?b#l}oL7Y#U=7RUD8*IzN;EH<(_yj3>c4dWfcYUC9`WOy6@iM?qMod0KLYxd
zP!|^8Vx957m2xZ}J10HbS}TKA%c*0G?FE=4N$=hfE3E=TEt|VzO9hJ-q6u}mu@d_W
z^cQV?EHrqRdmxC*nBGldx>;;cj?~qOlQ6tptO`;cNtg@Osa9Pr$6|igOK-cbnGsH#
zRj<p1IPv5lfifSyHjpbZfvpwl?sWjnSecD~D6;5LZT!7w9W=v>_A+M=Crs@yEqq)-
ze<cSs--)E@TT1oi_<96{OIw^)SK@D@!KzK+H*lU2gh1Q*{+eT;oS^hH$BE?6u5^F$
zt=gaUjA?6a$e(V+!ilXaySGp?TqicxJFLEOPPT!i@irpOe~uix*u=ad#NCP&*$taO
zxtN$-rZwAlf}E@`3Cfb7{}%bH#@<+F%S`b;%PN0|@)*Q{RnkqVc+<B6HskU0#>w=>
z+X0x<jk*duN3_|dE}xc4>ue8Jt!dTid^{%9U$TE57SHck`%)_-WjAgoFxQkty3R!w
z-WkL-9Xm9x+9gvRz448xqQ}uEQK;kbdbf;e{hF9@r!X_aZwP2)I$-x4k=iD|5wY%G
zU@juAbep1Ev@4R%2HdHC6pC=s4`Z&q37bFid4^5BfU3yQPswTieH|Zxk8j2imdMG`
z)XEJIh1AMmX#>$nDiwWGLE^Cl?28K;b36uqv@GMH%@fWPuATrzad=xXerhFbdBmqj
z{H?wCtCM$S6gF!ydXuTze>!Ujml7C2pA_N(;S~lSbzPaBz)~#k*&n(P5{*N}s(054
za4|eF^LJJ{@#P%n_JF!9vg=%GBBUFOONBbrY>c-jffZvDBW80PI0b-eCzap32jqX#
z(mBQ+1mJpet8NcbRne45@lS{NV@_>2$2=SYUbak$-xMsm2R=<DXG4z#f@hp-$6hB8
zb~+5eu=}MG0W2=Ey6ff;rW}`?0_eQ)!ZLg+6sh-?V)SVQt#)o<+M`}g-V5~U4>MDT
zYH{D0&@P6Z|5W0Lvw<jv3@TSXm$3Px8zPvRm*N6p{Ez3KxaK^<E+!UOT-|%^B7i~*
zY!!lZLX~(&xJx>EXl#8kl$$*{TW^mmrh&j?t@mQTOY`5!Ni3S4%RpRgy6N!P=yGVv
zAm`-mSLE0g-Rr^*`l^h``#)uVNJ!BE&_JmJta>$&<%9dR)m`<hYe7j^c1lC9BTSj_
z?l>&Qh$tARQFRm1u7cFv+z0;q^*IL5PqxLF8=+~A5o2Prn+TdsJfX^cjvWNJJUr@+
zlvQH=n}N(~E~&5=r_ui<j1p9k&6p#?qC1IhH2M}u7ogP+0XDl8#6^?`s@V88XlIhF
zRoGOYC#Yzz^k(>|+X1LAYN$A5dXA6KB)%Pe2auCwDC~>ZL_10L>1iCB_f8;}o1Ry0
zl0(wR?*=1PDIoW_m$3P)(UX%&HOOc8ffiGN&bQMsfBpS{MJsV4sy@*t$dwk4r#=Ab
z@?cff7Slx+OyHP<grBP+dIb~YD<Kr711){|LBh@fQ(f+sV&D5vPNKcD0sEr(MTjHm
zB_0)H;KM+!q|{<;gZkoUVa{8FI!T=~qxPYmt7ez@^W*tL-SU^0n^@`zFj^H^t&ej=
zIF(7fKh7Yr&yyL^&I6=Hh-)nul=1mfIR;xv&OD;)X%Onjb3D=U4B;%;jhH>7wU2lX
z)E$s9beCCBh2ar}Pph|d=yB<b{830MyZ%x%z69W!H<Z!C>LmJLVN@Ws(7~_PzCdQs
zyS)bBv}r_aS5VKTn9ID*AFc*^fx37|wDa<n^S1@Kv9Z^R_1}Obn<l->zM%-0qu4I-
zES@mO*JG9|z6I&@xjfh!i@uXTLCvfkL@C~`VT}K+JVd}dQ}5<@seyDn(k;v~P#kc?
zY3~EUV^*JW=Ldw%&91H(_8}BK9k192e@w7i#}T7q=%>&wBDV~p>odn3IwI~;u&696
z_tq~UNnLhgzkW$LCxS`KSJ1AzY=IcPD$(@~nA$)m825Y&K>@h&xc+wp-TGSY*Q38c
z<jO*wB3Eph;!Dwd%^^JF#P<*uL1uQIVT*A;{4P3$yX{A4a|0_Sbawb;{V7M8#9Q*Q
z)-TX*^};m0n1^@JAI^FLtG*aG*J{<CkDVq{nGQe)33LWnCd$?-)|m&$-|-MYDGsaA
zJZdyH{@9QI3xVK~Kq`XW`MhARNn<CIRVC&TQluhItvp};3Q_h|>dpU{vFaVmxJ4Ko
zWxaL4`~-`#ac^J4!BjyWy%;gtYy|PZLV(5Cs>?fU2LYV7yo$!*HJTcXijPFQsn8H5
zeG9`%+ftx3O`4>7m(EF`^+1ch3^Y>0dp&XUviUz=;+Yg@F9&Ub;#eWR5a}}3%hO9T
zmb<Ed{)PnsmtY1!n_<}NsA|^<AqONQwqB92#A;~#V-*L>jdA%vg60P<3%GFJ9^VOa
z))TmoK5KPIbzEh!{~8cQSLux>)`D`Yp%h1XPnOFLb#xR%!();N@^4^NTda;S`A_MK
zUq%otn9G55U&KiN#|7${3YyRGbxbTi2HG{J1E5;bp!=;rm%6(w1q_?qO@Lf^U6w+K
zcit4ls~4W&Nq)4`^Flugrw%PsRNmp6<?rz3DHHtNw#Z1j*U%PQZV6Iwzj|dL{w2^E
zvMz68%AI#BkV3ZP!~m&TW@{jr2RDYrrXrjSc0{NX<Vp2?f&NG~SUNmecN?HW%sCp1
zZA2GCD;2}#`E~&2doo8>J&q9OR+9sjV?=r#Ep?Pm1EiY2QLuAkHcA8eTeT16UiI3Y
zbK3a3@*MZ?-N1?hsW&h2kDQ8xl~FA26Xd$VMph_tn7{cRfMyXVY7AiU=3XE!R!c`S
z&h3~^m}@WD8`w3(j-DFsun&NHQ}J9L+nLyMUr<XQuN)rI2$dyebQ~$VDz|;HcL}oE
zXgzdn9X1Uu10saHxH(lp|DPtcbRlWfc)+3{9Zh^Ny$OIUeGV|H847tuz~W1Vi-yyg
zD!qNr7NFGHb;IK}5o9%VR4g)qpi`~1%iDOj09R!As5r6>$`!$vcTda|&6lHwMngA2
zXM)XMa<~153Q#dr>Jdnt^{v+fmPV4cn5wWdYwD%_V%v!TZUv4fBr9z~d=udR4evCB
z{xMS~X~szd<dTyBi}Gp3vXy8vnsF$8>w~1Yjg2wo0D>;A#vkQ%o*<`)S1}odqbB&_
zkQ~p(Y~}dhp+K4Q>=ds!0-)G@P!vz*2<W@QU4I0-Ejjdi{zx!au|ZFY^uz~8fw;{0
zGjEQqkAZfX*)qqiBK?8iu8oIk2;+xNMoY&N$0hfbDpluwfrUuRbs%>BC*+iy(99n%
z5^mrjaKOm~i)NRf!I)>N|EqMxISM+R_7*H~@I7@Z2yMvw8*#P5F0I~YZaB?<p+$=E
z3OmsbjNEy${j3adG{6|W_`fnxY1jC1%enr%2W`Uk7Z7#}*|}>aMqUV1@Iv<#Dsib0
z>M(pPXHq$SBi&VpFWX)QWY)HIvOQh*@=Rj~Scw+V&LSPm(wzrf0pvn!UKkR$2yh1&
zv0jY1GJnC|8G9%!%&r1*ChQUG;tkOhi3QrLR}*x*B!zcca}2u%&{Dw@wcZpid2LQZ
z%bfT^v<DTI@3Kr#i;lVulunzT-<f`WP6nSJ&6rPi$JaOHWE!|c5+iSfHowv{!Z^pe
z352GTuKCeh^1oc3pU6SvMz>}}*FIp!jzY{1=?d_<eVY?t6p10X6L$5uOpi`e{%=Rz
z;Rx7a#o#-01Rd@9eC8^Fg}k#8iXnGrnrU=cJRsVIwj0Cv{2mZDK?To|_d!twUS*G8
ziZC~kW90|(cPKVw9=gs$AnuoFto&?&w%}b4gS!M=#bVL@2(*WK-n-OOaOtkmZjS<!
zkw&IR6|N?QTEo_Oyk$58+>OaCBQz!~h*o_J%*nHpVSDyCw7UqNS)%I+NOO&)D%X_d
ztKma|<b`Eg%u(2BChep43J=Z9AJI{C`-g?yU!DX+9_ppQI`Jt8vkgVDtSujT8pQc(
z4cGb%q?^)n-sR8c|7FQ=+H;VkhAef*db1#^jli`Q4Dp2c^9s^a=IqV!=_Ylt`U?=|
zj~>P7Ojq1U^zXu}Ba>c<kQ^WT8lV}3lBpcGi*|*RmtP6q^TOywTpudEPSDLiu|5tL
z<yx_8i0|Kobg`0`k|@0eAn{<O^`64ch^{$02EPqW6?LUIhKq0&*$~KG(pm3-I4#<o
z`*q(7sLpBff7lrpewgF5qHgJkOFqiU;QYH4ahf57638BS-j4}Wjj?0nAO)RYdnc;t
zNpaSv8NoVE_BhLZmXS2m^G1*m^M=b9cz|T!KKpYpGh36Q>@3qc&|AXESgpOw7lg^U
zp&@Qouo&g^4l3Mzt@xKq!tW{Q7LvNhR^A)U&R{yf!p~AEzMTVT_UKfsK0f#!fc!dX
zmZN{jU(lv)C0*{1w*{L2*k_mHkH12@B+2}aMECg(sAx`Dvf$jSS4)%a?xItmImd2A
zkM=$+#N%Tzut=`@%>(M>QFoPEQ1oshsOW6Ixy<71+kSxNX*vqjw@>{6%tb|&-xdqa
zpJ`SWt#Qu+nbvDQvDbpo7RuI1vCKkHMJqLPpI9jlUKkAS=vAwxMF`Ux!|USo#R+<_
zDO`W29oKDw)pqoU#}p<u`}o6`G|c_#_@~0vx}e=m?;_n6fX><X2I1)uOJ@v)Ai8@v
z*0v0o`7Cd;mn;Wa6~=D4zj#fETNmR)^nTwg4`3;iCAz#+ast;EtE>QSPU##qrirGs
zTywc<06}L`nple2BB_&>F`KMJ*vYfxW`)szWdIjEc_`JIf1y=!q#iV~tucEbK<<Dz
zW$24}2W3cy4C3@Eg4`CvhsU*|ii#x9!V9emP;~VAcA3=n7ebNIjI^$v|HSo$w$Dqg
z3F7+U3>l;U8jWFHQ=BQ<mF(p3e$%yc^hU|};i4^?xME?8khXyr2z71K%`_GJi*>+U
zIj&@DzKtc<1!90o)mnEbgfl`}8>iPnx;?NYX=>(TJ_Rnt^}?MreH)usr1#3<`BRZ!
zi8Kd%(=HYq1p>=3eWyd5Evn#9588H|{EP)r)3J(LY(2sf-qG<d1<hLq3OKpdUjST2
zER5n3k^a94tIVFbcYOdelgWK#jMxy`O~qONl^a2sSH%qw@n!2x@;4N|J#N_yx~L%g
zX9kw;cwh@K%EVsvtSt$;2*a`PSaa+AwJdOAPtmR$!>2rG-zJ2*wP1WSbekNTIj}WG
zYzOVc<nLq94vwmK&c0IE#XwEXii~cw#;#!1#@2%=$rAN8q3X+MOTJLp3`lE|`*w#m
zhcLln4Lo2E0CgYiLiQq9-E!tdMl7}uuz7})+YR=GvecyKCt=1-f=Gh#^QQd>W`2)|
zo^d%glU%yawOT!p8In#P+Tx5x5Z6LtD1|X?Ij9Wk`l1`-7BdUx+iMu(uBwX@%keo2
zW?($1Vu&orSy+7#Khz?@hGZ2utBFW(Ni)gm%??kYHe*@|n?pQ%#85lA&3|(K9$T~%
zcKvkx5ly0t)|KHc?>csZ<WW*>!iae3{`oUp-F3wM-Ov&N)-hizT+ABse#=^F@g5*a
z*YFsrVCs>KS))W0xp4rDfb1-!D*qkPERuBF*%>|9>n~wSN7PM(D9Wv;(0&kI3|S*$
z<RmDk!Wm4gcA&pZuazDx!e374C8d^wpo<~Vo|GdF1}bC<gVIW@b_hVxV=(++z{loa
zm?L30!qb&E9|}Mv`^piA5p-#wrigM(d7f&G3k77M8XgzbBFNaf600BXq=v2<2OZ^L
zeM?-Zplek(Y)HIX{4-)~)E&b=1<P4;uw0M733477JvFimhIS#6C0!g;i%>U~?cScp
z<p{dMhbL<RP88YH^cYPi0;wY+0&gl@@U2q8oRt&BlQM)26@xvta3=$qbDbR4vn#^U
z(9|4lk}hkq=No$}kW0(6F+6j38gwBTJax?Ob-OcjLh^i;R$VKf^*gK??@XMJTqVr)
z#L@uqAM!5%%P+bxEYh_uT=ZN(7e3jXF`;iQh~^tRI(AafpD^2CkS>?2f4KnA+-G+e
zyNE2hB<og&zDfTE@qCgWQ`7MKt#FS9I`P22h^#cPya*H#VR46fPrv^_IY*3zInCfL
zp}63GfUXMr9FG70ehGl{w1<ZHrw|&1Z9v?uu%nO1{Q~aRYSGc6G~yfNu*---9Whni
z`3iz=7}gI~*o+tE6u8SxOHH~Gz-+<@iggdxfY;<WoVvibu4@5kI^XSNZWLW@fKF}#
zcH9U+8TE$hFADolG|o)WOWh1WiH47f#}%{;U=i04OW#s_hh@XY3j05H0bB*V6}mcb
zq~15~HW0>@^tk=~x97k8ic9P<JtLTG_aPUT3MrOjy%;D|D)#0(_zv*uc!%mR`PdmM
z(B&q3j`AkmnbTv`)`K?3-33BTxTv?=-2|O9V?WPx>ir*h*L83Hh#S?ifmr)Ke>7@r
zyr!UeElUBN(4TxiklU!Etb>0T*70DDhJ|R-?A}p>CUwgfeJS>Qh*+)!izX6Nzq?qV
zM<rgIKHy;}%X1^z&hsA0B#)`Zuo;kMaC1vZhnELEmXYLS67L{0K?-rfabhW+6hKQi
zu=HHyNy2V1H1HiU`Z;LV1s4Duv@@5_o&{D6(~K2|zX<8+TAPsM_KZ}I6I`uJQU=m}
zo_z@rZey0*_%cCq<tl1SdDUT_AC0#a_IJ|u;L(I)RbB^lz1$gjt}`Q?YwJG6FGBM4
z&7hH-Gj07As7oiy!4_ShV!3gr;GzyarTF4)NXo^oa*_84QZ6l;o_e3Kle8U2TEzY)
z)I3QQ)(KkA2VfQhZq25lV%?8{NVB;u?)@zP%T%5WSzZz3M3VnA4W}Q~FuX{#w6WEW
zO+P1|*-<D@>AVLBbZ#mE3Z*!!7SFFwX$-sRcVMAo<AIeg`Pfa2Hh^bQzRrmyH@56;
zW(soQbTuPA%==9(8V?1gW#7rOiRM<(2@V-vH3tOd$sF_G?+D|}Y=h|cJ)vS=?Tqt9
z6-?D5Kwd09CA455NBq5LwFdnJ1Ovy6j;5dUH`sqOi?E#Oj@yO$8yZ!bx7h3#psFz9
z%vr8j;9~z@fhkwdB>XNYY>wj1oK>{8Vdwn@=t5OJXi|mpTx(QQvy}-=2m9yD4NPh>
z03I|CVG2pttXHrY?Jy;X2SgW5H<=A>GKX074}j*ypt|^EUMNyhb>}GTewlh|{3w92
zk;Nb0YUYD*V%bwg93#*TkRBQ3IADRC2<rvpVyT4yEa-K3VjQ;!q+3?XPKKsZd@alw
zH*lM{63Z->fpSwHPJ0g$KuIvg84(LEk^l8o*yVo*aCC*wU;)k(U0`e=5*D>gWFJq^
zwcuh2hRwP~5laEO785WyVsW?EpFmt9ox=@@dzJ<;*GQ#dStz)stM22L%TalEjG^LY
z0ZvWt%poSHEe~Qb(IXa;)A?7(kz|O%HRnf#c<@gSdwqOfiG^1LcTsc_!`=EC(12C}
zH@*VQQl1~>fCCBX`7;>hW!cT*=KGbPT!3U7&3N(5%0LApbPS(tz|USK$Ezf#oy~E}
zKp=R>q0{z*9OUTwc?B(L6Lk9jrfAohi+H{8v~4vISj56LUQ^h4p?;r)Y8Mu7ygHyu
zjeSU3Va0F4%w-NdHe3UWMrT@UjRQou3OHirc5x-{734y20@55Wi>9c(_0cgT$CkqZ
zO@KEFA~ktvjyWQ#a!a@1^7K0Xc+7~Hrl9}CnhDeHctS`apggnM5nHbd;Ks`8F*WPW
z!d*&@)av=42(m0PAH>+9kWO5@V7z}oh!bbvMEa)<18}`Ktm=uIMVnJxz?B6EJG)vO
z{0z0m=n=&6@S)>vjZQSelB04;<Xwt$MY|!`V9Jdh;Qs`|wy|Skm(ltE>aD3tGzoNx
z)5#l70mt}@Qg=*JP;!>$y1`?iimG($Tzaek^CS5RrZbe+19I)SmDLod`~@08;a!Z_
zW&QjgT8g@O<!{hVPY*gziY+z)C`QTjLL=HLmXbo9WCQ*|V}VVfNm7T?3vEu=CE#ct
zp~hX;7C`2wG~`K9^PVv0)SHy{OKq7WF>Pz{#w;pK>9VVQSYaQCrn_`$9!U*e+M0ig
z0gqcrZ6e_iO~PpSswfw=t&?7`#kSC9ZiPog=*{m5p*eUIWSQ;qpR5PvYH@iXPP?0T
z@7JkCz@J2gwa^Yb<WHEV<R{?y9W%niXl1y(-aj&ewJ<x2xI~DV)X<WieTcz(0ZA%G
z#>EP{jJ#s0&hqg-Ilh~<6i1B%GE>tog8f;X*8m2;aJ?3vDr^R$vSD)irV+pfLu@pq
zsqN(o0rI#=i9hJ5x~>zyl%0gM89O&QjGb;gFoCeMOP6}p`JNSK-s<LUJkkp7T4Bb=
z`hY`=HYcHj{ihUmXJC`6H4NN;vjfm0P(ydTFRDmE*P+<I9o3n?;*}Yc6dKEp7g$sd
zwMsco>;fo6EID;T62B2<HX?~SalT=H5NB4J5Vwdbir<2c=cD3Z_6hLv%I^F%N)2vj
z#by-{=b^*y<g9xKVYC_>_cIiPjp+{WxwW{B_>WAo^r<k*QWsk7fxVEfYm2VS(h`RY
zDx?js?Jw72=rJ96yLexaD=zEBO8hh#fF{?Ykr($7uC@^drc!2oF$K&_SF5x1p{7F)
z26WaqR!>U%$7@jGK|VSUA(r{ao>5(8*~3B1G2X|n#2KQg{g|<F(Gdiz{aH)fj|F&L
zmaG!l8S@*b9tBK!d$<Vn#?gd}zS+y9cHeRN6Knukrs<kwDFz=8T=Z|wrT;0i=)pL7
zVX<<a5Q>2%)Zh~cI?1}RC`B7i%-;^<lzxMg^0y^=CgquVG6=0!H<pvC9Z!KUU+{F)
zgQx%KKNX0yM-7kh3YxuaoNP+<htt5EAEzVbI7Bqj%boFxf-bgP$<BK^q#L3zvS6+n
zcm|-k){SbL6IuZ|mUQT_y|)$%wKW{M_Do_G-Gm*ejEUAf3&>nanvhb=do~C~!F2CF
z1s&hUB%JFCq8-|q5Pv%-e~ax4*6wk~xgd0u^uS)+d>)i*g8nGBKOYj7u{InT%Uu9L
zI(h~9fqxS=Gf;c9#?=>Rx}nU-_~U<}U0dI3962pV!0e?GGcPx-rL%0vUbzCAoYGu(
z+*J^+gzs?iR^c^3mNCZOG*iop`}J#qs}h3IeI-u34#YXr1q9p(K+5!o=D6x+2-i<b
zxy^3P<S^vz3XyJhoj7s=-4gfT4(6K5Y$;Zp4qX%^+p(2n2f>~_^$2}o2XM(98Rc=#
zHg3*I%HOyX)YU8Xb+E(ii_PxIkS_Ua$ZZ$osxdXlCT0D5b0j^1qDaf$mm}dVLweo$
z9|W;jRi6;RqV%EsC2Okm5Xwc5<}aI>s7Y(=3=r3JLNAW4_Ymy@a0;zm-vvU<y$Oi)
zJWoKn9*8~K#pc*akh$N=)}}8y1R$E+|G8vlj;>4B+_{WL1UbtF>_6j+C-dhx_hztP
z`l<X)Qgbufd?v_Q=vaP8v_75Vu=3JF^zpA}GL&urQ1+dk1#-JdS0J+%DOHXTTpj6k
zcu&;n!Vq-c2Z|4hFt)&P;q%~Th}X&StPnF~7(1Egen;XSZmwDU>yqkOrAq82z}1z5
zusB_`Mo1R&OTR$a0+bXFcJ<Kj#he7I2?nV@)&OoXk;Ysv5yOqZ{TwU;cmzg}hJz9D
z>&pbyt(YmJ{Z-?`N5nY_g15;KYIZ>|V3gycH~7a@(jb25n~<)<#7>>FUGp{oOkma2
z#`!aHv(7spWY^UkYrdCb;{<}D9Pt4_ZJy%ZRcGw@p;N%`4DX{Ug_ni64wONU6USGA
zNOV$5jQlvK;oGt_(e|H!xmXq4FZRTMPXWx@<P!BQ(R|NKn(u!`u;^~gG};y4@Hr5S
zmy`8JiZF9%oc0AcY~eJaCwBM}!kmz&_G`a}be75dBOaO!Ko(rBoBtbvMPJp9s}ySr
zsaB~&Z(_utIY2HV&I)mV^<Ab@GaU6jv?YMEQu|LT$K^kPQdHgS-uq|5P86RhEd9FU
z+h2iPswR1Gi}`+Ye3suabgnh4HQ+VVSblCuXTT+1L`6e$Y&#FAo2WRVg}uVOdP1>x
zX}_Gz1Vr!(k;OWL8`muRsqZR(05k`1*e2oLPe|2jofCS72<vMu)E_P~Zq6}T`=kcY
zmaInSnU@%@C$n#xT&!$8e}>ZbiuL~Eg5dw?VR8F{`Cl2R=yJuNMS)x(ahD=C2=NHO
zu3QIuNqh$`R{OR%KD=0v8I=|^tVKD59=#MO@}M^;7#SX2I)CTa>DVDG=NR&+gdNRC
z%R{=sxwDGp%L@7LIxJ=z20&7NX2a1d=6{B9zjJSq{4;WRoT8waB|poZ@y1F3Rlm`^
z6U1S0gCC`SZDsy(cAV=r$2+3^DYux=Vb!eyP)u*d92Hv&BP(1ZEjNf@A+j8D>SST?
zYWa&wSr@;NhP8wtkaY;;)HU+|v|L}3r{xBNx!cH;{xFd)Rg?V9KPlSn!T>Kr?PJ!;
zF&gDf;kO~sMU`>2fyKZ&If5?c$B=cQ3xnC@09IxjX@B^!{&C`Z7M^d<ugl+Zh*gP;
zMLTi&Pg7@XKRkcJ5>pSgOdSDYK4{+40CmEs3?ZPr`%@oZjs~(cY0N5|7R2piYss-Q
zVvX^$5IBt4#wC9t=$^yYP#0wIg168H`3ss;KJQi*;%FF@^G1>;(u)MTRGO}({%GG2
z$gJh1z1G<3Z<%h8)yf|>`Q0b7IWNchg8WH3I{H?$877CNH2I;M<&WACbeXKYEr=H9
zOf@#yf-toiJ1Tk=EV7q}`01jZ7GsM(T<-5V29NiZ;<Bxv&9k(zmk}cMICNWJOUy9*
zcipueq^nPh=@DB#U3w=#cS$Lo)O(S|4TbtkUJi~CJ432)!(-sC1YH=`%yOu|*lzxF
z=*U=6!D>gqGKlk-xL=q*U~<<kdf@IL&JqU^e$2BH$L#53#te%k_9j@!63peiX&-><
zxP`tQ7HqmNkSoKrQf79}J>mv|{*c#DQ8w&74!}AuyD}`C5)ZG>Uo+}4i%1y93#2(<
zakMB*jWIjfrGcO;Ak_;Oa_5vlJc>h2Ekn6*tS`C7(h-Bl1GzmalUjPPOh`TW5Wy}p
zx_IAUi&4#hE^|ZT=l2r;%*6Ecl8W|^Rxm^Y+vN1<`sJei5t7Ub=>2v8H`wr@L*j%E
zNSC>Z{)+7H-UZ^k8Mc#rRN`%c)Dm}YPxcUWigLn+{qH0IxWRqp*tiI5RC!!sSDuHj
zm!AUVd|8Lb8wWya`J-X+ClQ4-a3vAL5Il$+3??2k#a^qh|EK-1RHh#-b|{#c$ucrM
z0rl4-fLs?wN}fcBTaR=iGVnO{pM(n;M^(S%asG=fMr^FG|KemQP88{+Thg<_xFbFu
z$d%?kC)%<}Cjz)gJST2l<LkoQm~8)b=7?P53+!ZIr>^_xH;HsoGTbV~4+5xZBQK|a
zb_!uv0dZqn9+#d9BF=Zlk*7JL?n4bei?F*sh6<x%&^h_T37yRym}0zeu0QPSh>r6J
zx<VK`GbWber1ODX3N+BHTJ9F$QgSL7uUwG-pU~3Ib9!Sg25^R~%{pU~|3XtPyk73C
zpm|~APE1w~6jqEDT~x)k37h^eC&C4o$uZ9*(5|kuO?2RSF3mA`&<@R;Xvct8db56;
z%RyXn`6gkIy`lyrwZ|e?0$764CM9vI;3pOkF7tY9%(;fJ`Jt6QDuVdwS|C@yj*Y|4
z*Fid4c?6+bbO|XK#(Im3{{k%Dth@W)nE$rl=9O*%FjG*4>wLa1?iE%H2e>A=6^at^
zJVNYyTmHPUk1H(kvH;gXTQ&M2jkd;gK$i;3k`~khlZCic+%<$5OWy@Rze2?_Caw|T
zHlxvb-lIm>jjiMVyZK*zj1}Jt_Yij3FgsZHUMOd;9>+d|?5`A9496NqlU_5v5o*q%
z7l?K4&*`w^eDnb*mmnPk&VCfSnl47g>0;x9GXOpI7DoV9xNr0rIK^wh(dRn~BW}D%
zwA$kYUB%HcQ-l*_R_bKvU~KqBpxam0cxYbZgqc8v)Nq>xm#k0b4~tCMT750hvBuZS
zvDK8P0L+!1mN@tsC|9)${r}&eg`|A-lyAdXgq<Y=cG^=-6Xe9ue=#5qd=a3Kc6?{F
zmEsB^j*@1b821u@WP&-W^m6{wTl;twfWqLI4m<KEUV}1g$K$p-o_ie{4zZyc_y$2|
zB;6i6pS^{+cybu0Q=<NFWoW#{Pel6uxAS*xJex!JTlO6gc&&?naqD~eZyrn3G&)NN
z#g#2m=Ldvco>n!&MA2qlS4;fxAyjp-#uk~uj(tlU{1LF5ucAxLNz8v1XgSltLoa(o
zHnJ;zlB2SDYwMFrHU@kK=z3Y_S&4yy%u`-@X8n7U02tGS4a%1UDHDz<d7a}PQAIuo
zPx5F~iBVsJx*m+IW8!7e)RFhf;#Y+W#`ZMp=K1QgbL=L#7`uN1?QYM4OKK#pV$R7}
zeTn6Uv~4Zlf?1S0F}uC7MviCn{6eHl<`+Sk&nj{Gcc3(e91F)%-$S_5XicNbBFQ3a
zm~d)~vzEDjAn2N)5o?Q!M4Q>H`+GUOWK6of7OAa_Sls-RzaKd|W+-S*GJ?}u7ySYN
zE3~ql@oP>DSLQ4?)tnpu21p4hU`zaIp21aSReF=P%nTte0P9L!-lijN-4D<llM!RO
zBsX+EFz4LJmYP+I0CKLN`FwqW+Mj9L{@sGL6!5~D=C|t>u3@sFROudA9LzlD+#r4y
z>Bbx{BUxNtE&*a`((813C}LMZmST?H`!5Ama5Oz3v<x)GlELqq%Mx~(yVJq@Zvvbe
zyX<tgYOCcPA5C+dy#iqxidQBd=x?|lJB8I&B<xaANG<7B{c}b*2+<+ZTq}XNA{y3*
zh$JP>6t7UwUvvFNU*EJch}o+>X}Zz8$|_(qlQvgzm<ZQN1{*POQ2qc-C9DHmPnfHs
z=N^^r3#$Q{F$3%3<kg{yjv+mEN6f4tysfBB#`{8?QIkA<y}l+i%)=XWj2@i-Yh(o+
z>x(us^#Wlf-dzj8RUXE&^{{mzsq&QGXdOz>S*ImeTih(fA2TvC_&;0+;K7H3F3i`F
z2V80`KLXfk_MnJ7Z)E<gjr)#$aiIVgn5**`-H(DMQ#=u$s32u;>8p<?MiX{2dQ1A;
z^&rhF2DGFjUi&XV@H-vG)`=*@qRb*4UhW~tyd1V(oF&TX>3ZQUB3(Zjb)X=8s}_Np
z#AWLffnv2b$Cev8$a~<oY)r5yMzTMsM8n^JoG@FNI9g;;knTQh^P*u>AgaaH_ZYt=
zVM*1P7`hcf_gi+P9g{e`EXOo~#6^!BqqcUqwR2(&-JY;>sbeoWYX?Z`j<4wEn7AWE
zktGTa^|?Vi0Xc1%<TA`~sIW&3#LHA$DP2LhL7)dk&1!`=^f^M!JFX5avNM#W9sQi0
z+26Jc7zLFAu6I|$l!naf;|>w#tkmt)xpg-Xmwg!ep>y_tbZYpu(iL@f(83(08CA(a
zdqKKTEMKcxKD&30BoD7E?~^0Rr59aFy8Jo&f|?l>+5RC2kL?F!iA#NRk#W$p0n&MT
zeU6P13d_uLd??7}$BQVh97cB|h?&8DVZ{<u(TT86YmQw76auZMMEZ}<5!ijAuXtDh
z?9sakF}oR}5Ip8<I>3f4`Fpiq99N@J`Zve!6KenCXFG<sLc0Jxovl2se!l=0fVp*a
zyxa~=)39udVHLv8E_p((#1%D!y{m*}sUG4KENk|m;Kmd4XL>FUCQcVpL}F8cmL=|-
z1Oy+)4vU#ZuoLfnlR-T!uq{G`#05{<<%lW36p}4m8n<2-T};LZK}DB}OEKes{IL$G
z<@-iQz;_=4T=X%HNVhl)k_yU0-hm3cT#RXQa}@&)2XdWZs2-K+65>cr_1GXYQ5+)3
zOk?KfG_e*19RvHDN}O;cQPfw5#ViGtBlpr?RM_%c-9*IC!dyT!!m;AfkWL(@BHX(=
zHh)qzA&bev9H*p*f_@e4GO~0|OT*?9fZWp3Voi>Jo|pliu)|N<nF5>$uEludR<zeI
z+&fBd=cGN-%_oC9aRxWIG%1Fk0^}l(K;_w6BkM*EjblXm1Fj3FZuE;Fcz`K(Y%<m1
z3L^H=sW~CuCW>87gDi#~Ou;d&m6GaoKqtqVFl_>sJR^UG4O2WX5@wg1V}-K_S~Tq0
zGo5H%cs3}#h&|ZQ_(4QbA5M34zvUM}6i)`%)6Q|2#b?~Cu<O~unfAXffOKbWs*nEv
z&VS<0Hukv)(#dk!3Jp<TGzp?yXjlHJpqm39VKmvJ{{<+jTDaNfPWdH(l%pQA`u(Qm
zFF5yOq}=SX8o-USQml1(2D)3AS7T=ZwH2tHLo?4Geg(Lzfhj|@UkOPyIArCcKU@Xn
za<_2H5}Tc95>!Y=J;#ezi*~)amplFH{BNa`MeNttK*QV~S>`Qs9Yof+aN5k9&=hsZ
z^}y~{Qm28OTtnz*=+xptA<l*C^zVp*cX*|I;Rb>(A-kMp5%h+jBE2$oD#fceg1F4s
zOZCO-H$xYzEG=tg&QveVanSql%86(vEc<)C2Xx3SK(qkHHnI7wge^-r43X@WI82@@
z)CucIOouUXzA(PRTs%fhCv0Z7qlP&74oIwm2gO=<7Lm{rd{Tt#@t3geUEr=u(o1y2
zmqJ`F3=h#<_QgN$0kXJMy)af{dJXJsz`qcNIxNNo`r~rPgyekUg?oW47hN@%g_T(0
zK5*yEk#IfL|KULpR}NM)fWo!49|BY5m|S0fgfMNS-35LWW|)$us4qSeMT=pQa{prl
zU0T_S@Z=tyX_LnRi`-ZTkn(&_<Y?&%nbe&zQlN`qLoGJT?+bIoaTw4OSI>l|7%0lv
zv!C%4R8{Qt-Z<tLZA^Z%g*zpEl~$0?KLh4AMNDMftn-LNpUp90H?s+Q(SJM-<~&(q
z^~A7Q&=$|;-dOkrD7P~AN8_XyA<cA@-f21h@Jl&H7uQMGeHqdNz<3U5n&ZkE(3P(0
zPIx7MfKS@E?NvzFIeJu#el367%!q|jT>!b4JMdlH6n}mL(iP>_h@U!|_Ga<<@EG+D
z!KyeRO^o%O@BR+bQl%2lz6a#`vPbWWYd?f`eJlIN>!Oknw8y<45q8~WoVwn}ko5nw
zk@@0N2p5GN1!i<~?7O~l1hhdNJX{0e!qD8ESRutfW&@Ek`e7Xp#ZjUiWfJBklAVjb
z0kS}8{J;|d*GOXypjnFN4~DYqzXfpS4LtJLtV{Dx2y<dwapCH9Pb~REhLReGPIHSO
zSE&IdOTQl>&5H)<c6Sg>TQK!~t01*aPg)%F69hc6$J!r6S-^Q0j*Aa5>t~Rn(a}Zp
z#oINYaf;q(xae166kNxeD|Vb~txQYF*(%z_$GkLc&wB+q8$Dh?q|byn8!pyR{%>nQ
zQpLwoa}$FKjAdgu4?!~tgBAKFEzr{sOhY)&y~nYC$Pw_-i**3P`1pKaE@rtMo6cFH
zb5+x|(Ld&|FgW3YD5{uVVTFn*JzmTa3^VNE^8;J5+8Ux~0VpS(cI3~CM$+VKV$6d1
zKO8gQi(WMUv8s$`79w0spd3Fie4|jfcriei3p-_L>sMSnCxHf`Csq|rRapv*jL{-I
z8c6NTE7KCTmL))|^35Z)OF}v$od7EuPpt_w7pt#i#P&;NOnwOZl-9<70&{kw;x`d)
z%r3?NvFNh^a?@dCNzWX#G=OV>F^djAeil+lFl$oX>YryBAexhT@{ncozg^m@A!NHP
z59G4R%Pl+oO8l|Ezvyj<%@uSGr7{`_Znq5paUqi9fOuYjv#ctBeq1p_n1iO<TPOV)
z$jz%m$?o)4NF~k}OqDT^8x{Mm3{iC4PAqPdiPOMU01N4n1B~u?X&^|o{~^#dk{K-i
zGKhcOOzDg?txaQdp)O>pUj89?vQm%}<{FReFyC7>N0NgVuF1!Q)pBad$*lJj9YW0m
zty;=)VGU?+;lsE_fb%OnxW+fD13ABRD9G978X#0&zIkH#H6dIpEVtqzk<J2lZ>)sp
z84Tct(&<52mW;dn4rA?=-tX8~7>VjpRU8fERAjX|goj=MT!{3(PhWg2#QZ{G)GO^b
zBk!F<L7iK2DX!;5`qzQEdHe>1_Ab8&^n40~(vA1D5g0lQ+zc3x24CX+>u?}fiLEw0
zbb}E9nzcs9Ulnw841TeEV#PWNsNk1u<9Q5Qh*>1pD7%k_ba~RzfegmuLV+$ghSsc9
z&_3-n1`L)oqkoGbf5{Q5PTynE^}#%>Fl^$+dV@@J(lRzat<e(kXx@<jDF{0}Ifnar
zV@MZ-Z3sQJrP3Zd{uQvOt(Ft9rvS#IP8z#UVb?^;#6SEE(pA;^6dB)Kh)Yy2d+`CA
zLc3ZTspyD239@<qibX$u>tdP^b5Xl5$@GA~1CgtiVC!v7xL}yChazbE3Up4G0gu7z
zKc}IV*Hk%EkfU%6#G8xTLDO!${MfmDj*UiGSEl0Goq;@J<6d~#U7%bwR8$&Ne-`4p
zbBf3lPLM7XD|_s5wBUbEpug&F?(1o8k5hLCsS1c*59^6D1W`5ZPd`)GAC$Ce<z<<@
zK`3NXQ!KYHLFd>?uUUFO$ZGHHWkSDJfIpUUjHHho2U65w1osE>2N+Yqfa~h>2kmTR
zdb?sogFi^-PVt2(GQ{3bwioj?Lb{6SkyiE#yjn0G&=u0zpE}6ff{@cTY3q+`f~fX)
zZi;CQd~pkyi#5JI)}N45V9AP3sxLMdWScRj)n1vjVuMT4v-Jnu7&{`yw&u_Lzz+7$
zZ5b-sAm$h3>^VJ<_eTuU+riAoMs~PUJ0S}Nj1GxuO-&F#2__boo22xaxPSf#v5j{{
zR>h46Bo^w1c4IVRkJ%j8R{$t!OGDK45X`b86SOHp9Gi1jGW%{KfJd+7UQGToD)H!~
zjFveFW2lti{bzDc4u|Tx|JSb%#FZohs~0%6j<8(7(Q)IHOk>~C84rs#6WGtifP*rr
zXH#z%=^|@<MGtqt!64>RVLF`d$(}FRO*yfGi3C)^?OyIyC%dOZ!HNNdf-!O(4xj;q
z)0syUb}2b=D#t-b_&+h}vLgvQG3}K(nCF`IQDEj1^AV1hlQR1lp)T!shB8*boBb2S
zrKLq>c9~WLj|nDg42EOqF%HW``|}Dr+2lHe8JCgn!DB&PINb@2=R{j7(W$fV6$i9!
z*z-94a37Ed0kMCP<AJIS?3L&0`Az^ar!cC1SyYjBC$3I<PlVQVC)>@-MVMt814{9u
z0QWrE*!Y;M-$^+M)R-e<Z_#9#ycea}?_D8OS8CcnpW-lv7cos?7Y-{D?NDY5aSimk
zt7<THDi9^&IHnvMoC@LMCWm6ziFRcslw+1Cw-o9+&52a^^!!ELSl&sWBAWhyXZdLg
zI<v-3#GkRO=?oy(SPoFjvBa4GE+XAc<=IP!v*bc^x2R&+spmm3oSgNX1LQK+jX~!%
z-?`9Kpj)=hJ6#B2aqq4lAHDy9bn;!=f0bgT|7FCJUmPf;x}snorLnCN8(s=p<TAEy
zL>w%-5O0*h<yhhJ90AWcn6!v?1hz-)Tt2-5AT>2DTe%9N;6*(h9BF!85Ur+}@WHDc
zX1I=(t|9E4@pxE{o34j;9d!3fIzb_P$xbm^Zy@Z1>roFhQId}YxtVY;g-)8x7QG3~
ziE{O-GaeP~&$#d@Rc3rF$Q3}#%38a8O9n8A#ZmP<w*inZ79w%;?Ky!$KNC++&+*%m
z+o_RvI=)mNzbfq1S?W=u5qAT)T9v-`2HY&4CB$vVnSS)$2k9d6s$nwGTjl=zja<Mm
z?c5}!I(STAq@anPcmT}3h>LLaTrITrcrbs;Xd*MyJs!%Ravp(s2*NjICYXy*-`yB<
zKM7eBK`xeQsj?t=AZIJ{JeB`uNh}W!u}KZW{S%XKC5{(Vj8S?=i>;5W^<JsP#GP7`
z>}wu;nwV~+j>h`>nEVX1D?$gt!cI%w)t&`(jd3CG<?mY0gPFNWFBYf1m?LTP($qwK
zUwswK3}>wG<@MHU-^{TJ+s{f=-T`x-a$RicIpy8_>BN?9Jydwodmt_<+uqK2?0smu
zeP3@heL&EK;aCaQpC!N<Gt29Q<Z>bAYj1}yiGEy*#N7+|t@-}<2sR(hQTd3^G*qSS
zum~DLJHR(SA?!kzuyWJIm`beqIbcn7R=5=Gj_tnihn(od%&!Pj4c4gIzQxO;J^0`N
zezUI~vyGAQN`+ksooj0{sUhR_mU?&05mFsViknVYTYZzioY1R>U8Apt=rjVk8$E|O
zW+&$6yf!M@a>(v4juz=QuU=+}Pldq@8GyyeZ}a!;0EWh<qFq`h7R|C-*J8BFc=!U*
zt|>Ph@Ba?U`8A>T{-s8iSgo$`y;Ijnq4MwZ1GJ0N+1gf*-i}wYe*)8(#2Vu7KNGgZ
zCr^%i)rD2gNxuRkBV&fgPYSxen1+mx<$r^A1M*aY%r!7hXcg#eD!u7#FK$rJJ*0|h
zZQ81(^C`kCP3(&&%CGl`d2%$rltJs9_XjZNq0J_)ieUT-^X7<ncR`e4)uzIV#?VV(
zhs+P{W-YEYNG1G(P%6v-8$T)R5;!}qLo5IUNAZ6oJz$qe$G`wkxBp^~g@I_mJ~sHV
z?4l5Ej#6W+xdfDJ&O(PR7OVazgb^K!s_yu;79qX)r>flXCnC7b8TRy$@=gKbc3{xy
zD6#aCj%u>tqPJjk;cF2`3UsAeWen#@XlSxymB1{$3<PyW4HD}s?Cy;ZMdppSmh+!F
zwi&!UVOKL<uA*<w7UW{FyklaCCHiLsY1UHrCLzwe4Mj@aFWTbRfFL|q`@2auV`mM>
z2~1)&Nnc<1&mb<8o?C2)vH(ilK>M9m`;#*XP6Bpcg+I++Y-Oy$Uv^*(!|?+{a$9UT
z2+XzdZ9Cp26I93~!$P`9caSjGkdtHP?AU%)5SZ85DlP455QsfX;=0xI*K!`gP=!W9
zn2TOwq{eV!4FHQ9CS6QRRIwE22z4~tl9^`FH33{WJQ*cnK2VTHydrrY0X7fLQ5jv*
zJ&8kp2lbTMhTt8(2I=HSnI5nfA6A88*qB&)2z0Kg4t<Xhg4}u~z1Z4NE+o4v8g`-p
zi*<Kz?66M$oXJ}|#M6YR6S1P$d0oP8^a)s^+$hp%@gN*sXk=XmIIyB!2M+^q=}>c2
z;)xN^5?P&3o;ec2MV0Xel_`$`anmIok^0O`Va^060HoP78o*pl>Sj`!EvTrp9{Q=o
z(PKbdEAFd}8e5|}<DT_$Y!sTh9(?xtAgUuOq$wNZe_4%k(<3esf^?v}-{`Ldi-khE
z4rZhJ<2C~>a_fx)wt^}OSZ23Oz0VR<y?IqQZA#mq&-F(!s<H@T(xR)ZyLARJ=PC}?
z9u(+0%Agdh&Ly@1F>^5&X^m|~Q_fL!(O&xt!Qt7TZTYL{PWDn)i50iYpO^G(+QXs?
zMr+LyF9>jJ59d<nJ2etN%h9$y|GGR)J{fpp2N3daZ;T%lgrQm_Y`!C5v%@dA*2io?
zh2U`U8b#nP04|>v(8<lfAYsnFkt46~MViMv>RH+q%83?kBI9F0MZd2OnLq9hX7=Jy
zs<UmPBx<F9WKfT*VkpVBr$wNP#M_CCKs^GS9Bb{eG%E4yo?t~)<eiqiWW!!SZUlLH
zi*su91bq3$kD{H7yi+%-OKr9{SWz=fF6M}I5*llxd7n%(&X0(5MOW*}F)*vM8}|jO
ziVfaZ)1-KZKxdV1>ByLJx&1Px7Y}tJ7Pks>dX)~D$d%%)`W&sqsE~}*{?q_wj!a5s
z^y>+5IU6Wwtk4KeXJ)+tyF~a;870OYC1^<i*A0#;6D}l+9bhG{9}i##RPgPvU~~Sz
zzM%or$)yE2F*q<RR+^A$Y5(Jc*8Dx?kIzMtm$X4Ew-I(;y<PGnyLLN(yR1xlT8bM^
zD|G-<_+*9NEW+$eCWA?B)6|*4?42%d-yhJ)YlTvYCA)L1Y?&MX5Lj@S2c%`ecDW$R
zsAYboC;!cCm!87CTM*JMTO!;Q^g<Qo!Jz<}6*&B*Kqrc4m*jWgr%6DCAhE)4o(xGI
z=-lJ2DTJvXet)}T!2=-7gX;0UI>P&ciyp&`^E6XVKM2f8qHc@-9-5=baYDBq5PkZH
zjNvz@y-Qk)`;W|@F(}7%M?ty*+5>uy`$V97FrGALi^}7fj+G8P8pN%x^Z1x9n)rV2
z?PpQuymYlZFnTP2`H>!YMvON<F2~_MDU56r;5h9vC?HOkAm_ql-X6;z56!66)g4<a
z2>;}R_s0_mn-91}il0w}EHa~PdNiy5NkC2rKiT`ofK#AdK$aq!IR%*ONR{3rO?tWk
zr{<)1O&WK+eX-hU8N%v3Z8yfA4&=5$XOPaeVq0P6YN=kn<26}y3v@2kX9Q%Lccx&M
zMpr+EMBf?t<Mb@L)-18^nSibq25aenXFWmw5E~3m7Dk<&0VzAM$oN+V@aV)gYpej5
zl~szafi%WL=YqL;v2fQrAdd)vyFDB#ey4D?gY({C^Lfy&M5zU}S~4Q2#F>JNggg4|
zB>t=OoeUe-So1={mMD&$j}S#m)=!C36r}Z#a$RkBOQh>0ab$V$pA0l4oxuzNZf^Ny
z-0XjlE`3)or<$Fy@+BZHF^ju~I9)VZV*o~%xKfleKtYa_EjA56-A|k14@6XJJ8VSE
zb(R0bysRhU8VJ}TN7koa>py6*5pjhGFETNx$MhRxKi&hq&e2(K#j-az+{CBfDQwx`
zecH4`Yl;PL1f&$n0DF!ISCLEqSR+BYimdZGI8<&FVqw6rg*`4WpWg~Z;X29fV}+fA
z9VuxD`O|d3B2m_w!$dk!y&ggjyzUOC(2YxyrxmW&D0%3UwxoP#M%%GTz~)j;4Vr~I
zB~~z+7C#W8valE1^De>`3@-nnc!%Ye+zsaPRd{vMy7!arseuSQ<{U`<arXkcwGc)&
z)0^I(W1$zEXpPPN0?7|uV7#Sp(J7icTe@Syg8*<=uQA2$4>=}Rp=0xh{YR(#D{Uo$
zB$z+qkVgo+c&v6i5MO4=+XND!2g|MAM+v*eXjf!-+J6R!bM<llxdL2mc4^I>rOwIm
zjxhHDF6H-s9Et>yQ5<~`u8JLZY)_a8?c{jep5^*00-T&*RwU0~g%zc+l4kBFYe*^i
z@Lf%a6Jqbjz1#-{(DN9|o>$O9)Z5CrE5k$or-3XZn5D#Yk<Qh(d1&qZo&|HRIJuD_
zLfvy9E_@5`s>G<5po<pJJ3Dc`01FeUA#~}<N9KaBfI4w{0hd%re7G=wYIi;R3!w4n
zX2av`+81emq1mkcH6pkY(qKr?KNnPG=CEP0_UoBuUDp;TiiQPT9p2(Cf-VE9hGa$>
zr@Rg3GH`#8Lx5>DK<nMEm|X*4D%$IT@9+r?i;)o8=~qO#NInLu#C7ih!B<Y{wT9dN
zgB%0Hie&eA%|}4yQCetNu-_Hxs&Hyvj?YB9ecJK;u;9lgG2YN?KFyy%!mMtomA(^!
zc=2v{Z1fpnkJ-t;PT77~?-xvbdMzeVUqHCj#c@RJ{3TG8k-SVX>ucyj;BZjM#J=?G
z{1GdM<n`d+-vC+2Iki}64ph;gTs1=1QUmmSW+fi`HYY0mI#b1Z--EbO;5S-$y64}1
z$g%W@4qA<2KY~;XH*9olCfeo4{HZ}|pyhrBqM^}X9IPPR&<rr<7sA!vo9@lrDa35m
z0R7CbkS;W5>&!uQzX7;$>}ocx4z8bT?dn%0#^k@w4VeoxGQJk=Z14)MYk$Yj146p#
z5KiOibODrVB4f%u3cCzaeScH@HHqPLpML!1>^av;n=T4<$h@Ggu)NAi@-LkasLJhj
zF6qAd#|%``-L;_$0Ju<e@G`67^@Pw~m}wvduPy}TM7gNJQ6zf#g)_uzi-k;I{9RDB
zszXP|WYNWp#tRrMx)8u676Ei&xRb*Ibx#5Q3fB#-aqObdMLVNZPb<at#lc)YI+F|?
zrU{{Z9L_TzOcw>mv}oI83Bt}57c*1X-;?V^f!5k^Pan;TOvMnOQ)-b}f=(>UI4}k+
zMP%oOp`f&t#|Uv5F`{UWpKCM*5~Jej;D5IgiO0mHA?ZQM_~x@^@_$^%i^Z4CNhSAz
zYl=2Ma0J6?-Jg~N`2Wj63B|t4=XjGiXQT`@07v#*51)GlVwiC}^o0L)nxsKk(S13j
zQC(&Uf>$_ikEQ$n{%?HD^u_*yTvbkInq#RIp$o-_UT<ticfr2261V)BKV0Vq^iDsl
z3~9F8bV{C}mRTLp+3Qk)gz^3wq{o2;?KHTS5|%G_i!?`intSP`U#yk?uWx9K`PPAS
zJsDgYnFn!AD6A@a!|LL>S{$@yv?~=~f8Cs}?u9hQdjg6vi6?W0Lb;;yPMA)9liR)l
zb^PI)G4QYkY2w?>1QspM;M63tdJe*oGWI&jqV$Mi;7CI1Wd{z2aEDFqO*&A^R^p|R
zpe}l;6V>D?qo5g{rc8;AMiW#gWIctU{d!OqOIE;oNNsZ==2Szn${F<+0Q0+^M`rax
z*m@i0IFn!mM%SC<WVj)u%ZDwSg17{z1v+A~Xs5!?C7rr_wgphZbH-8bsMN>yTY{M{
z%+rX3EEMkC3J~eidDV{F5Uw`L(6MoZXiK)vf;t&>wGDf0TYsYS!v5P6c8!v!wb)L8
zD~)|UMzc{C;#4Pe(hm+6U2qj`RmHZn7wrh@yozJPG&n!B6R?|Y0&BfpcZQ^SSV!Ps
zeOCxqn{)SM`cStUkgJ$RXHM$cV!_=3T}Cb)Gq{WrP>fETnz5z3bFcgzL$mf=@N43}
zfV3v2o4TpBACya?rQYaqkWN&$jJaYgz{1Ij<StRgpeVBxX@>3><|;DJcc9E_k;92M
z>w#Sw6iw;%mB#!rDtBg1+WbEyFr=&(6Voz5n9!uxo7NZ);d0bt;9ic;1<;^c()DX1
zY!0!rOpcqI1vxia7r~y#gz$kB0-r1Fwqk6@b9FPcE5*g8w?q|uqf?RlYAqnnU?StR
zjxF{R#3y)K#HeykEkX~MWaNV3z)m8#jief3%44*=QlOKjDQT1VM^{cdZ9|!Pnrjed
z;=DY-gBklfH@r=8h2}F+FbUVHvlSE@&@5cvO}Hx8!$!rcqMex@SYm$Iu>#`ek`WY}
zueNxv2h5zMmoYZ?_GW-r&oEwmUVv-Qs|oU58%s?Ds?PD8O|$-E70@m;w|p9HG>#6K
z1Y9lG&=Ju(8QLPoIz3(%2`5Jmk>9g7MY+%np)#tutS=*oY&{nvj-3MH0?DO_M)WrY
zxfX1**kBGh0Dx4JsZ|^+%70_dPoj4n2;efYAsrvP9Rv*@wdv@phH=Djioz}qD_?E-
zcR3uyveMNU4~z2ucqQuQ$Y=K>K%5m*k~VQi0;q|o>zPO8KXF{cGVH$s%nLeqd@Iu9
zwSA>(bfA&mKH6zu`4s2<lW-xt($1f9Y^LExM_eJ=ZPV4+rOWpk&HjEIs54^s6#o{9
z2$fr6_2cvRbz{a1jpJ%G4S+}W6Zl`O;h18;iG-`~*V-3=<eC!heG>mzw3x$WAcS%D
z$zVmlgcCaQSm=}t>C$;Sda`2$nPGa<xDu;Q1)yy0-LaX1t}8A~OR@i{&_yk1i(_hJ
z6GP0sr}3{!$}P5!6t*ndvdmkQ4NeDkuf&0GQh@So#hHL^L{@Q9a`rnbBf45mKOJ8~
zxH=(S^I<|5pBm&i{B02~nAA`|{tMFCun5J{Osbm~&jYOX18m6aE`W9s({(6Yu*G$P
z&5u&Ki(b3OMF3_aNhKxTb{7M=#CY=RVz{|U2z941;)VYbF1RLBmiqrer|E>|9W)K1
zNS;LlSCKCTa9WMJy3!GUy&S|6v_C^<%lKsS-|q@g%O=I>;3lzm#=XL6uKJ0w$(00M
zQXFe>K3|R#1UWlqGp=bLcU1;3TxZlc{2G8_4yTKy6NjPK=7d-Za0_sJv|X3K<6MFo
z?|D4{wPJb9yJR=yM06d7ajQGpZv>-Y*q<J&pi9wS)>FA>-IU{3nq?#dBfb>oHmv9H
z;GJ9ZH+WDXj}@i^m@Qg@Hn2*;-SJSNW*^-}@xHtxBT0!@iH+|DDdvK9nZ!#)+9J>&
zcAz_CZhi0`5Vsp!|8jgHIyXGax1QMM0U#>PC4|xO!h;YVitxxV?ja~=B0m|(J7e!z
z4?Dgb7{wPN%xU{crqahflE3024r>{Xq~cD2mdwJJEkk^pVdsG5yXJM~((@iCZXt>-
zl@6Bb{}F;Ha_utaDC{iLjeDIjE&T+TtEHWliusZts?}VE@w?1~a69Ngea9yuNe*ui
zar;va>jYtqrwO}FwW#Fkser23!{8auL6fLf$8pm05SCuNa<^2lV@osg9<y>P6Uv(Y
zT3-M_YB&`?Q9%;W4l(+_m?KIL-5URU$^UZYk2kYLST-72lCi^6*Lqf{Ymc2<S9<Vq
z+ABF7ZTQ;bwHhF2(`d?GB}Ty?9i_AQe*86{g2~v(5b4l2K+GgAlf{Z}L*}t%WUM9H
zAIP|uDf=KHPF2T_I_S9fU7%u=DskZ|J!drcdw?25C$_{I3OiMphRN**&`y=ZQM?z&
z;~!=O4_NXiHRYp>;9x_$^m#u4aW?EPSdPhT{-{p@%^G=TL-5Mc?=vuG$1qfh3q%(m
zqDSD0AoaNSbHIYrzQHhs<aYlOl%`@GggbQ+o~009Otq_T9K=`p#}&X%rh^AFulpK=
z>S_hO+BbwPQO&9Sn&$wxklH7vb=_&g_@I%4?1jE1>;k6MTo<O2v6VnG8nw<8iT5so
z_(r2rlfqRgr9KOO5A8e<VqLFmi|d3r4%U=i@!XI89#M-2ej@DT;U>lEiTi%d5X-vc
zvAp58oC5wx>m%k~r<x+By2IlQ(Jr|!L#UXC&jV<gLh%;!^n>*9(%cohigG#Sb1rSa
z_7!G^VHC|?f2sf%yo0L8qoSP+yNME$2?O^5^8&gB*@d*Q8Ial5O7rEkc!iJtHBNvl
z%A<7TSBhH%!N&&F({C#5^2nzGIwh|8*3!U96K*E{{6``o7WkOrU1kwZkP(<IBx17w
zkmaG-UTiK}Fh{EA#XN0j=2{4d5uvdyzF#>1n~Yp2(=Lku<yjo1tbNctA=oufZsq0p
zC*D{T(AgE^^NhtnTnqYSa_$ftFP^c9m?I@iq4SmiGp8Fm8ydI^RRfp}nLOqQFzfNM
zRUZp41?|Sf-$I-!(v`2EWYs!r(4T-@MD&VqNbl^OSOeL;P1J4&Tc99jG0}{SH<yNj
zsicE*x@93;FpYV5h*$x@b)49lW{As#I0`qC7$P?84?r_AFvVU9yIft`B{Se3DX5T5
zF7h$jl22(sj+gFXG9n+fQijqFi(TRFD+4(z-tb|;`jY^ce9-V%XCM?AYpK2Ms)Sum
z)~b@OSA|e7{w>8bt3ed(RImPI^-MEc*T=<cK&Pa-Vu3XYJJ<9EShNc8$LQheV(qn|
zt;5mE0o9-`yD;KhsXsg`rStRRUSoVJJPA5()z=~HhGa^Quh)e%cYF>a?mSip=*nPb
zp}Vu&49j1YdOPC;kySa<=?HVydBT(lSDSi=6L#G(tQO(oGdB0cog@4w3%I)YRD|=#
z8C64Py2WzHNI-LdshKvsv<A>o*i;=qiWshi&fH!UNq%~}GnO6=;ryh<ixWk=uq}E+
zL$)Q?3bUMGG}2aL3;x$JPJ=PC61%Pk;XwtDc|8>$gRQVZPN}_<17OXZ(>DYwItJ74
z2O^z0LjsO5MLTor^WoI;8|6=!!(*aI7^kJuYYMwuXgSiFt7l^%bI9JJHEo?DkP6q0
z<ynoPf33y9!I1BIJSeabNX9Pe)cm)MXlF!0CT^M$3WsVdF5V2poJapfl(#kqpm3a2
z#%Bt<F6jX<iOQB+<RoNA&vZF(OAs?eFLqua(&b2dP)uL80x2pqxsg4Idg*Thi+LDZ
z4L$z}`tH^_D%?!7M4v%tpr;IX--b9Ya!02w?KKH-C&gIe)gANS@^a2`)J_0qv0ki}
z{<w8lAeXU=<rnQRbvF<<V}q_RzPv}K>4L-JW6@>>Zg^Qupe$|P6U;+*^-ha)38xBn
z;u?_{w{O@7#3iN=;I(syeF0n~+Y?t}h9G~14wMcayN?5Lmc{tm!*#s;At!k0cx{#-
z8ccV1XEx-&txLryq6Fl0Tcr-_ju}FVs_MY~fHJg8$G!y#KTLp2SK<g!c6ze}ITh|+
zvyh7A#)FWp-h&$5>@d5%*j-_#F6U%%zi2leKI7wYk**X6?a4qaeiY{Xv_I7;`~O;i
zT&86B9=o*WIB2oyIi+~5EywAQ`;%yIuK^rX@)o!N3m%=E4QCH+|Aasn5Sjzr&P=oQ
z!{NVZmynzBI=sLmoG_;Fa#MVzpyjrL(Hwd?$=Hrvpma~^oML|wPN8~HIM&`D%(BAR
z$->ju948Bgi%qD#uj?jUZK>pjet`-IsiIo#i)kXv5(W|WNjvxC_z011R{UWih%+ws
z&!yN^Sk(!9ak0R<j72Ba62!qlH+KmEC=AbW#PGiSr@RBRc@w~&l7=Ow5QZJ>EcQIG
zfVm0&aS?7}X*>ELaEd-&-b0r?7{WDUbtw_sQHU$Ui9$n5oO%d=woQ%koq`qzPJlRZ
zXp73BIr^kBZxhDA!!m#cGesIv1K5J{$zewlqiAMMyW$nmPPLhHdMs6A<zs=|l=A3S
zoYc<}Y9?xfl{PhN9tY;)>S|6~^gABF%~?)fm6<eInvNFiOc^zX$JQr67dhBeMuXh7
zLfs`W5s6h!ge(NEo5w%&8r@i7&QP;TIkpnuZ=^MN{p3tCfJONfNDDeTAJm?*w2G;K
z)f%_8DcRcvxqLlx6*BEqXo(-Xu~Dbz{}6`W*ju!Vg>>1_>sDcINL>o(L^E;*2<_I^
z6(=a@{L)DiJA%G5fr|XFAB#gp`+Mv&G%v4qHV7r)?XNgiVPuB$>{2Xo4g`aGSDLs6
zpKA&W1jdvh=b1vC-N>UKqWDbf$JkwjbHtYb3XB?!WnxEscs~ETR=irt@eWh=xC@;?
zNqeV$|Auf8aUG;PBIAV=ZNba;-Eq-HISM18+Wo5kfWU?Qd*dzz-4wnAtDVu#7Xw;c
z(2>hOUpltgTQDV0hbw1^aLF;w!foz<p`9()g<G0<C@DuM;VdqexdhrG(ZEfcA4R%m
zW$p(g^V#{Qfw?j);?>7@ycEQZlx{oF7q1cIj94Zh2~#cua7nlkG#p3PITibZih3cm
z8q;9>bOmAO+tkZyPK$(bSAtpYi!(HFVb!ZZivn|wR=OIp=tKEouS!h32H3^X-QCvM
z{8|8Wlup{_SCdCw59pf7>n=m^0ylwZfa1>ZAcb8^?Qj`IVhuq>7ip0Zzcd892y{)+
zEa8&(W@x7>FZY}ipCqKZ!e+im{ea=+JmE!aN$E1HMpNp}SmYM|FItcFI*yJ6R6BiF
zj1%cK3x|C@qMTZz_O9pN3T^(?T!_Suw}HA=Ec(#4^~EDK$kugu8NZ!a$YSsK_^*QM
zQ0n`0Pls?GEWD|^X!ylqBmQ&;VdvM-k`A=xVdcQPK%E~al-xlm#lP;(P<>ltr#3>%
z+>;@O%e0u@{$3!91dc&DRY2W6Lzr2Mk5V0qJ|@HkYSxQR+uoNGLg$p0Kr;nZqhOYH
z^8GoH&IXy+#jipvL`WK4o)s5L;h_%!R~g9Bw)JUmKMd+}X%9oM>WSqZ0fR>rydzG2
z6vC}m>Xf_J*2e%~66PZD$Hxg5?BIHa)FO`y^2Q9)=fbI`7MOp0f<H)=(=INCh#(P-
zmrp5-7_h3q;3N9age*GkWciv#QU(ZfrDX;ZKh$U(-nGW|Pv-yY#*7}ywNhx8OCy#>
zv^|~zvalon#d`@WKb_+x4GCEddJc$UYGD}fh^Tfvo#lQffO1QV+Wox491X|66}Bjg
z*H^y)DJheU$nh^h6mrkB(HTp=0)U`%X`Zu)heQ<vgI;9xj(z-V8SUT&!Qu_$`QGuD
zw*Ov3lxvBh2&$`cd>{x(?`n?~-z8{1N=JzO1XF=9m5Vi6{8A818H3`8_xz9CXvNy^
zLzvwxCEDY8(JFA4c(CpVQ1AdQ#s&&k$FCOdrsfY3rLNfbLui+<9WS*q9UA!&kh#Hl
z!Vq}c#{gouyqqro352^u@=VQBP(qx9oV>{dWV#?HLGhRu=K2(%7&Z(420Ea-{xe{6
zKz6Bkr>ewqpMz0&tm)$a$U5sN%Zuaf<1PtF;ch_|SQmo(;;spBXXcLVD063acXqMG
z3GM-c1OfziC%8lK0Kq-M{fCG5^YqiVFXz1H+&{XW+uxRIudb@TKw+mdDXGv-`m_L;
zFEaUk^mzf8L+SkMd;tJ|Y4U@<B<K#MZqAN`wdA?43MhY{l27C8<7+@y0k7uLbBgx_
zxzT&Dg351kFY|w(8VPof`fes^AVzU){|*0o-bIeYl*;v~_D&xPcUo;7{arS3jEUcY
zx(#J+9`An-P06v9{zgG~Ak)}Qe<;zgz-W*AMCV?^N}o<1tNaM&Mq&L%03Fx<1mYTT
zACpw6M*a+x8>P3x;`5i8*Gjoag8ibxPBqyqtKDax)&G+~G|f)fg++s6AkGm@3A<Uu
zx1ED<-Y6ham(sdE-*`^otgR$&aNi@N<1YW<f}s3ELC#Q2^9Okx3Wls}E%8DFmE5se
zM(nlNY3`D$_Kes7$z%y8-=seln`a^IuQ<@?`ca1PvR`YOhrc`qCQ+9IL^^+@G!*EJ
zc>&BUni0FIq4=*LwEzl5Uo2Suu2tL8Cg{Ci0Wl<SGl3%k5yTzfVt3Jn2<JY>ZHwP{
zJ^we3jnI;rM}*rd`FFvNPe|rH7j^Ki7FYkaU?_)Sb&TcDhCCdheDtvMpu#S)UMB8A
z)O)HRug+6D;_FPwB{J6$i<Ia(QA#6r^x_3-<;q*~nx&QmGGoy9g$v6Ha4tz#g@)gz
z5eJoYZnh2*paItsGcQHZm1jcZojY}dJq5bH7>MbvaMaQuuJ1rQuJd(G*T0NY>6sRb
zE=$m*K(b6nxI=(bkP;0%+<rL_s_Lhsz8B^GXXi_F2U_*$6@c>`De3086``FjcBZUj
zcM5O>)>>ZXUJ05;)te--#>x;>Yp^5MQ7~64SrqGzcg|J(|N6g+eHF~3k1K!pcH^oA
zVNR{!`DesxAm$`%N(-;O$K=(4a`u|jcN+#x^J*`(<QjzYl9S|ONl1LonxL*|nqSh0
z8j8NPfSvy&33+;A$Kg&&&k_DjVV9o5OJv1}m}_MDBt79e5G|t%!jn6iLI#E8l>>{D
z$H$a($eecoH0ahJ3uvAqfs~5kH~<&XHiAAhd?46`C|izLbUdK@f$k=7{D*PX?+b>T
zY|b2XxrKz88J;?m_Di2oXWiA+JJb?S3xI#gh-MwaW$w{|iOrM>ywtk<;XyKbTr9mF
zluN)>j&Evi7UpyjrBI~@)(3FYaWCJcQvP*=5{In{!+zrp0bD<fciHIbx^GaR8+fqF
z?Zn?3G(sCJ1~w|GaC5_^<f8^q>(uUY^Nop7&sxg1Xu@n#AUR^w2zybGE6aH)9UV|l
zVy--Svl6$pqmAP=f+9h#9a24BYPxD7fLVawD=TBJy%iXo#4$k}yKVWv)6A<4pwiWc
zqvxjWiQ(?28J=sc({=!<_tP{FBvq-icPdG-J0srB0@UManDurh?5d(^nw+d1FUTFR
zzcMKn-UZS;o+3A7$*yqft_953^Njo%yOp$1SCHU5P7>s$xkt(qI0i_sI3BVmcpker
ztALy5?p2T)H&M(sd;Fmwm70v;W0!qEm>iJf|E!=B(1H)w$2Wo0YoCq;bi7*3+YD^Z
z`;@j<3xI2kh)bRU;@fJ8LpNhD_V?BjhgmzRq+BP+873DkcnEC+p)`G6aY+Y3v#f@h
z?>{F&!r(z%6TjO@IQQ#{94x%m1>l5`Q0tMI#kw86iHoMa6|UDJUAZPD+I4z?-KQt%
z)ssp4`4jAhVv~NtPEeMYjNGq<<b^$*meb;SNUfy7S-Ce(5Y4=s90ZOYfN(=46Hq-S
zqrN;tI91U(^*4irT@{Qu={&azFe`C;g@Cpeznu)^eyt&q7MWv(xqt|$<WK8SA^uQ1
z0k!|PQ-EAU-QC9AQ%e-i3YgZ{;{2f!g~hccHkbx&u5+Ik--@gszhx$+n=ewjYC5=!
zog5Ct`}+e>CuB$Q#DRpJ?KIv%i~A3PcKI<|=SdH?)g3bm#{EgPTlWZu915n*x%$|>
zhe0?8B;fR)TMh^Cm_d-Dm(*zZS&snBJqQ^zTZf$*2<w%u(q0qdl+X)r;R)EI0P5vQ
zCm{SI9Rub*!mSA`_8c36ojy9Q9o5!o6XIEymp@Ku{ErM+j=TPF9RI@zsW`oJ0^z(=
zq|>FQ?G;Z1q>EPv;|c|dJDDYPhm#1Ct}G)4;tNqO4nA>oqDs^7;->)T-oo8VCC(J!
z=ITV0`BCG4WU#pT9ypc%U4C{?tf+CI5VB{|-u^TP(Sy8R!MsT8@P^hR`=K3A2c<!}
zrB*rVOb80g3uE!3!nv>MzLE{sD}u~awBV)3(GriJ+rZfFBpdg=&jZUP)}RyVlw^mF
z4M-dzh+La>tXll9B?)%I$rD1XazVkG5hW!{jOGi$Tv05mbdKy5lGiar)Lh##B38Hv
zF!!onIn?~*;_{vL;56C-mz3}7w+Fm;Au#tLDpHMMe+PgYY|dkeOUs{Zi`jk6a~VK=
z>gtUrE{8O0@@qlNPgnZ;X~_EbRoLSt$wN1~s>IjSnN;x46XrNg+VFiP+66);OS>L=
zb%}s$wehjxwIxEeqFejv0$hrur+`{u+#}4Lc?w?5L}~Jpl;eTxAY6wY#`NqrK)Rif
z=ChaWkG&d@b%$`*aAWzX9hbhli6p^x=?OkB!Y#x&K^rKRy1As8u1D!H#|m;kq<h<B
zxE5F50_vh?kp<)Ij)F1swrdiM!-Sbh)izEZmy326u!kUZf%gPD3+uF`t^H#E@Tck~
z^2N8=ozUj*04gs1@zz}cE<UzSN@$L|L0l@kcV%!l38Py`@%Cedodo-n(ZhHY9>B%p
zo)QV*{f?}wqZbs;Qw5jDh)kGe`yK$Si?{5bK>I~@B@gn43nSZZGG6r|r&Z%PH0hC&
z7BV@#g~<4vCeWq9?GUoF%A)`@4*EmYI9h}gOlF0(Sl}@bYN9K<`Tt4S1;=Y|d$fpl
zY#LIg-~SVm2OEBGn6Vj2Yd={)O=1|oV2HZvQ-I7roa-hjNEV&!yf%5-VWb=Jy236h
zXE`Krb3X&%{P7Vs5Wf-a{1HOp4KX^N1)(xYCOIG?&rY09Bz$dqu8~5uX&{al?T$}h
ze@avy8a?cSbwf7S3nh^(&xz$<1ah{#xWY|Q{7DEkpjXDV|0Z1DWzc>y!=Vi~eYwDu
zX6^;h0#$o`AlPM;;fDItkFNl^+Q}z$eEKSYnJWG<%~GyYUIQe{W@IZ1zE0TXmaZP6
z9lV6U>Hl@MoawESK-$VDHRkQ!2CPeV@{U#u?qA*k_qb?6jLB9-7s1a6HS2qLgera%
zZ5~K3iizj^cR}jhw!@=^|5HAK1-zXgx?U04gzD&g#RnxN7|cCmto|WLorC508#fE~
zOoZ)G+#}NY;i89>7!Ds7h&v>$NAa~Fmy^e5#h61sDM&I6?~1XXf|%jDrD%`41mwz9
zv9fE8H9iB$vmo8<IFa>b8GlTyD$2j>i}GCy7MyHvw)?Um$$xq+?)X2a#E8^$ajSm|
z=~gJRAIXmn5KhO`O)(lI-<4#fCr-Uq_yNT2fn%l4m@b;`(8Mj|n+m%=D7BN=w*u;X
z&mLCe%^v~X^~a2k??pLEkxaz}GYzk2!jZo}?iXz?=yvJVS<BzD#)x%igU)<RTm9<<
zn2+o^WD9n^5Jvsvq4-2W@<(1m`DdNOf3)KranACUtee;q9XuBZqq;*1hKGxAqNttp
zB8Zr6ZV-(=j`RZx&o$4?E5z{%c~D3{CT%8Hng`6alr)Dai#FOqpjnB*C_7HkuB8mZ
z=A0ig_cN4nDk!`CW`UASday*g?XkQ-=ZC2QiuG)pKmA3-bkQ*c4>n(rh<SiXk&aCj
zS5Hp}b<wo(<oQ2lhXhiTn*82Z;|;|zgK<p)*M|S90CWmw=nWPE5L461X4c=7zegmi
zoD~H)6SSmp5z&P|+f{z+Z;&#M9TjH<6uj2(iM<v!(JKAlSi}UXPvhgAMIp=;_Ge7!
zY-kq)(vFfB<Q2@lNfOPZFS_~S1@vK8h9p}r0hn89Dmza3ss8g)U^#6rptyO9s|2}C
zT2L*2LZtgZP1hYQ@$k~+8+oSCK;L;8ASaEL|6t<C`pbd2)8nJ#t>vNUIC#H~F)I*u
zlG6WY+1gf!xy@k3@I3)JSK_eTovJ7lD@$S?Sh0M;{(%PjKLM@@remo|H&_Ws^ETb%
zxRo82q0?f&Bka;L=G(B9ABgo<0n0s+=u<?xqqg<Kg=JUu=hez!OkACC-Y{wZEX_c6
z7D980SwWe{pA+IVx-m(M%Z5SI+(~GCjR@DcP4@#aum*tVx%zW7@`QMfU{^~na#);W
zp*4Xt1`#eVrf@EH?_m61qzi#QgeLgwh4_1V1`^kn*n2pTv+nJ0Vq&ZcFhAI6Vl6}A
zZ}<h4yp%Dw{bxi;YXC93{NOJ>5=doXE?iw<mjzFMu}S_50ec)iioeQapkqAc_~)3C
zoP0BAei$_#D08LS!$4?S7l8C7?|D$+`mn`hAYJ=CEzo60F4E4E08?VbdL^Q*B-KU#
zwLV~;{fl2&7_{gHB{H1>4+iSu8wzx)h_dN_lr1&{%gtG-b;eo^x~YRL;QFE|Dnk4C
zPGQ%hvr4gI$&CSMdzlNxG9vu17IZXc1@NhqKt5MEH@G!)V*gG2aq843i7=OYFrgmW
z6w;+aV$KU$qDiU^A<V#L4)^wS#G{)#%=?<L<`#sV1zjwqIykXFoZpi!%>#m5N>=#x
zj<{n>Cou+_!$)(tf(CofR^T2FYS?}jHD;Dmg}Sv+%*x6KF9~#-%p36KsjUHA6lUjI
z8=7@%Zd2mac)kf0Z~?AjtJLeh7VZ34Ls+wuSZKTL%4ZyOcocp9?E&0|d51C-O9-PG
zH0{Kw9U#ofDfIZIcZ7s<a%#4t!Y&q9e3*|+js5fAIz>&`iN9S9uI)KWye+^*?CjuW
zj>UI|&Uy2K8yeZwxKtQ<GpCP>@w-5Hpkn#Q@({ZUA$u8awJGe+y(eQ~K1`Ui=+b(1
zex}ivMe@}i|Js%R%@u4=d21o{tlM@kDIu}Mn+_?Rv<Fz8B=F6PyTRtYKs;5qcEog1
z=F5nY!{aWInHOjxNl3BAAAl&UJo-=kBjKDHgNO9j*c-&9m+#+n3B0DToCAt!jTqgs
z>DF;(zkSPBEpj6y4U?OMk&kZKr&kEO7z}E<H*=#PvxM{agqYL>ohJzVy4Xk6kU*D4
z%9lx|^N=vtkX?S<+zhGe;9KfFg<YK*yJlv+t6M<K*yQon=CSt)b>2wPJEOZ5nhK`t
zwm40c8WVS{<J;=tu6RIUr=?91R#MXZzO)^bo<T3{iaRrr7L^@4_{SYhx9dIe2LUb~
zf*Z}E4+|kHGyvl#g<Ys7UG`o!$^T*08h`2}ock;il+O6Bs{kCz>BOgW19%0Z(`V1y
zwHVg}T<?r|A9t^?oRL%!hx(w+&>B~bbL<D{hOqZT33-+knB%Y;nbaU#FkN4<zx<2U
zd{lo)VrL*ym1Lp6vPReyVXx5Jic+Y7a)A0qfSELD5Q!5ncMcYW9iMItx1L;(3i|)_
zrZ<FWZje6MN>d<Q18G3XBRG;0fs{gOGxH9W2w9Pm2q!kcRvXJPuZ)>a%-o}RpP3pw
zD#TscxBNV!Ajr(AFn4{`h{5(luYbhQ{v{dtum(A=fwXc%339U#Hw&WCj(A11YcV8^
zq~rmn{~$+VT&BI-G|_I~v{9Fbqi+u`ALb1`6?t)nKV<XD`M)jhJOrACNT+uRA09mv
z0RAxD#a@Rwj0Pem+z!i%^u!|w*E=v-)uQF7g5bwm#teNz%#2oM-uI7&tPh%=W}a)+
z{5SDWpk&4%h>MQNVPue}9_uhRY;nf%gk3c@OPpKb(Y_}XEInQz)ujn1`WsEtLpdx_
zZ{;L#7ZwX<oqzr;Bo7A3U^$M*E5g`}cQr@t<nn<hu^4_T2)zmizRhvF2&ad<1K(5e
zNd6l|jLvu?6J*^JGo4Ocw;df!>d+MaEP;6tqu1UU$DIM7fzinYp~7Zc7kj}o&T1r}
zN8pnc^gQPQx`*&s8Gg-0J3)zghT{42O9b>Fc>MdX(Ah9h*I8O?EvG+M|2U~MWM;jf
zB*h_(^Wm!k+|99jO80GBURWUM)aU`j{sx5V7nh&?@mK?D!X>9PL-+d|AA0c|8@G#a
zMX`m8O)i3T|6oGU&5<_r6NQ@B+*!-%;p;-+m5zXSTte7oqx&XV&>DXSa?_$D)glwi
z_`5KD!eDB@jIeWU#+mH=mzNZ}xVRJV#t3qjlGk#m8+%1T@=KKKTnR!&(v{|&BJxC_
z4K;&m!K;AWAUKtx=KTW9X5DDTtD;GOL(M?!dUc7d_dK{9xwZf`**?Uy>kEL;8V#~f
z1US7B@*XkEjnJeg%kp?O6C);|WVhQ*{9}foe9QF$J$SpD!Q3lREk@Z!f}ewKbrS3|
z@jiYV1fzj%#l+hQ!accl-s&HOT`wLI$6vtd0xW3dcKsKR4ENsIh{<~tJi>EV0XTM`
z68Y5K0Iq^wl!~84J8e`CX!{@U18_q$GeQ|Suig)0PIF9Pzk#Isfda8BU<Ihf#Sa2y
z-i#kRJgyh*gr($Ji>C!pBg}MS_D9N}$eKqDi_rqS#~Cv={{ASG%h0OFqMPGWA-My=
zwSPSZNm^358+hDdU31KMf^cr|-WEJoiH4!A{jube1oH@IZ0`FMq#H;#i%IM<+tXlP
z#94Z=g!EutNw8~ze+#C$zGndPps5YX`$R7<oxcApXq^GPium($&}JKxXOd;!_5zSI
z;F*Cm)^ynY?%%+!d^5gYUxISwYXfLV55&eVgSY^dirxx6>Xj0OeUNUuZWrRd#pac3
zOS#CO`BgyIv3pQ<I2@5)eyx1U<&K=atok}g&b=MGZBy(lxjlSHl*`gS*rNT`q3?jW
zmVA!~op(!&rVjM?`i;0KO}#@|<L*L5qro(jU-TYf_ufH?uhF;W`i(X4=Kt^q!%3Ez
zalwB}!bn6VC0PA^5VB6U7gX<E(Jmam1W>Puk3Il#8*>bc2R|$kxCPYi;d3Fmn~jS3
zK7uMOHZDF9?SDIaV)>67U-zf`|BZyWMY$f7I#2xblLATiL}m%Id<s-j9UI4qcI)Zz
z!5y{$mz!xys+9A820}M&YLDRxI*&eLOK)<w&w<>GsQ*lsjciO7mg}CJ8Aa!pAaIx~
z>{#imk_d|jZqPUa3GxSUTdULk4Mayl_7}5#O&n)b??f^F|G;v4%K-7QZ=fma;I#Nk
zK_|&Rk2h=L-fv3+*=iOal;455&m%9wKa9uP8-gh=Zc*YJh0P(}eBnGd%l80ug!B~I
zDnFEenJY#Pi>C!RUldPdg!qgQw|ghsnA(q!?t#AB)t={f|7(D_{qU0BDKO6hI84AF
z9&^*rV3Z80d#s+rhzH^kh0TfP3hP^SrV;gGU?ixc-Aprs@Fm(TvDPevU50d8=}{Su
zo3(s`rk9NNIMUW7D=gqrTI3dkq41+{lH%?wVh+M^BN--NHYbGBWkmORTe{ZVfM#GT
zuUg<?Y90WyNDS{m(_>x`+6`^IW#%if<)E_`R|qis<WFt%{N-cHB8`q)g*ZYJ=Zy^)
zgsgLddjCT-1>`i}854dD;r3=U3?v6~EJTM1cE4sqr>0%8-$MS3<)I}W6yYi`p&`WY
z{|!LdxoW>C4)z788;m)%n(j_!Sp>|LWm(k|q00(!d6|FWbCE7D$NfsQFA5EFb<Eph
zu@bd~u6W#%kj_<}bUIKYrf`=Ftq<WItP{3g2EbXeIo2CXX9{s11B1BedbL4!Nw;FV
z<@nzP_k};n?k~*WaBa~O%PwD1#k*XMZRyKGoH2%E)PJ8909-Si0>&XCnX!^JcRVSI
zHm>x>+A9*wLpk}SWZpSdn6pdW_0^T2-BMD*Y~l7n7X*v10_-f9DPjeYWTCfm+TsOK
znyF~MRSCjHseVsbov@RZB3n!q?W8NMZOk^bIPihPfa|S?`vIjmX$>$s52BqhF@8;m
z%xJD#cpzAS3nN+4Eo(V4<?W94;Sg>Fv{khqUUvkD)7QaLSCG*wjAB-XDsl8k!tR7g
zb(g;T{%HR#KX@M~?7t;Z)ta-@SRhK5cDaj<Gl9Y)E=fgrv7j&CG9Jnd=+aH=?U`0O
zGA~?s0{`a@qcbx)lS2Icw83~?RK5GFhi$bsfGaMqx<~vT(nU)rSc-O<AjiNQAifrv
zTUVwNJ?lc3o{VSKbgTHLP&W+j5+?WRPKV?-To2gAL*z`EW3}}`Ts*E(ZxQAHwc(`i
zJ|YChunzK!(FPEiaWs>hI$hs@Y8`mor<2UIVTqL`8!?|C&t_Kg&>im58v)YCnvo@M
zLXbKlu5P1Fi)`wD@W-+5W(3{Tj6i-AW1h_mrt_B`B8=Y(Gncpni|ra@H<|&p#urGF
zq&mmLLR;`f?#rmb#tx$4M6&UZJw=%lJWedD;Z`9Ocd$9O*^(fA4L|pBzrt=&Eo0nE
zZB+p7JvfbQC%{RyV{+Be5o>M@l6%o0-Z8EaU2j9)an!rtJdV6g`L4CX8&teACd6#i
zJFz@{M7c(6TR_||^<orAiem(7>LZ=?6gF!*xyiUoq>Gb0X<*p5T_Xa0{27tCbTW@P
zX?tkbP*PK#Hy1#k(4I8D7vT;xV(s{B2PpEbRj0<(9SOS+rg^J>Fg+b{yI^-BN%onG
zV!oY#NDzgYj<{lHh_WEin+J8xy4fzEE(DK&aFx&^K;LL%^(PUzW47xg+Y=Y>T9V2B
zVYR~DA(TeMa``0%{f)LYQmwvu51@Siq1|;)KE`jkXGu#emMj^@UoTX3MUmr<y$HK_
z9X)dOcDDeh-kUrd;2u(t8Sh(^*!T}1<}?-{ta_rIN?TVitIM$hoXTKdOX{`{3(5nc
zqs4Dcd@PJwP3eu#6@)EV(c+owj}Y~qhhHV+`TK&nO)#S7=_=I`Z{QVR7er^l_?PI+
zPdnR;rJG7xOdL|qSyhNjJAjINEw*besUWW&h{dZV0y{dM<k?IB<s9gblNHR|$G}Y;
zd6rfn_wN2G^6MnRzek`uFS=84aT}y-&m&vhD$>#~3Uib;Tn^vc4oO|K=cc5wVh1$M
zq!P_g*q`eeV>CRuMx6xaY;`4&o|cZ;I|1DZaZezn2klBO>;`p>xh_HY_3Iu0($>2Z
zH}6N-JVeqyDqa%pBy<4Fw!L2ptq;bYII|xzk7##7N_2xjrOG|je-w6!Ie&8iMVB;c
zV9p7JitZQ^%@Aa{VWm4xlp7t54$X9r32`ReW#NNLYxI8x0m+2ZTwFMrunUC|Q9Lfv
zZIib5oI`e>0^}?w_44)jsnF)c<PP<qy@mitZh$?~1q#y#^{hx-EW&Bao7xv5bA{`E
z65>D80V%Q=oz{n#u|G&=9;<tPOJlwRKuI{=y=^YS3D;<fW|1x-7qkPo*eo<FKqXE+
z5IRpQ3?v>&(R{G;3{Z-#%dO`QA?!M}54FIDqYedd4n4@bH$MW>O_4-b+SSZ`B$!#n
zE|NDdhGL{33duIBH|`hV#$bynHya-aAxGUZuX<GZTH{|=RND&5jO9X{5;g#NG;WC_
z1(5zU^zP%RMhe_r@v1RuosAEW%Cl3S{b)XL8FU_^3}*<b&jM1^?@#{uJ`?Pze8rKm
z`=6jP!?MF&DtziOpejCRpNYp3cEUV!B<JpT{{k5pk}=uX<D9146|~2WqRc`bV6|4r
zX2*l&@yB&Jwx3k;$`b(HnzdexC|?!L%+WU#KQ#VgIP^x}iTq`bu^?$nbrOJ^t(}Ll
z<~teE>04r=_b?6;==2#3IE@!gwfb?OvD+zx^W?__DM8Ftrvl|s=rtDiMW=%~T~>9j
zza{+I@(e&1hF-{G`stqwz}}l_;*&E0oO<F3XNJF>4dm20`17CyP4xG_lxQ@MWT&y^
zIVCB?lr-!W4FLWl9r{^79=JTIz>AHs*ttNosa|Qv1N3<%2{~`5#zyA@xRLt?)dABz
z@kF5-zzA)kMFgzD`DI+Guv;9Ty?XHV%LbCxqFNmFSK^oj^fi2nT`M5>KPi2%m|p;5
z)=8|0(1xBeLtwo(CEwj22q`^~Q)W7vtZ@-2g=4r?;tCOY)N@%FKV1xMR&tNY7-v8p
za!J7ue`pZa;#FZ(3Q<7(?(YsynKC6VQaHDLQg0uKn*_N&lG*pg)0YBJfIi6rJ{D2$
zUGl1|33u9M<rBHhtHq*M0JxawTB6->Wr@HE5T)F?uPOj1SCqFW3&_(Z2B5274M{V~
z$f6d*Mdi^UHPeZrT|_B1@}{W(hRa}kOum+&*~)#F>={110fh2v`HRJGgmC#~@W#ae
z<)13h>EpY1LYyd?FPJ4;V)U&L7VB|A!}d#_E3{91Pk%TmoB~Lwigf>dfi9Z-UJS(w
zw*zDjw^!57dhUM|NOMtzL0%P9F9F`ld+sdBu))Q7*f#>qpC$~E7r(3gkM<NjeG36j
zV2ZTG4ilXx1w<PcH{v(9OJU;A_c;C(aey=W@O#VmBgT&%9nGTYFufe<mcEa0z5B9b
zkP&Je@xU)W;D~$dgV3I1WwVVmFDcl5F4)y)m68vHSo$F#lGY>Idnjyn^`uv|M?DPS
zax-1VJ0fX`bj>mOQ4>iwYM#d+NCnO6cw1qQg7lUvHHy9d3FeHk*h+lg?uimjo)s#*
zKNs^p3Ff+S0>pbwI=c&WUDzL`8mnzrdm5N5i4`NBC0tkE?U8;A8zvUje+YMa`L!f8
zqFxi|KH0`;htDg|f#e=7d*P~Rmlk_r7VDunPY`{Y(Q>1Lo)~Z>B=62T+&&}JtZGA?
z_Q}5>orxUnauf8!^Cd!0dsiIxLWv+fe<D07#0%h<anboAl-q)<LAElly;MG6ik1xg
z(pNy>Ko4ik1zsauW~)~1WB6t*p=J1qQ?I=a;>OA9)7U}_bM^bAs5kiy=v@7dt~C3<
zCa9h~QgFHxO>p&Fp!5eFY2((n%U6;!spXb`r$E>Xv`fdJ_FjP)wk(MQ@xXt9oOZge
z;E73+`&R=c-az|)gF1H|^geOnQ4J-_miR`Lo3|#_4ETtriVq5iZwndr13oR#jG#d!
zbiMjRkUT_Xoxk))(B}Twk+JZ{P-c^qr4W(#$GXBaQYhUf3cIzr_>H|9(X&-7&o+KS
z6n9(M(^grI#HK%d>QqK?vsY0V{z<_mZWQ6lqyuZ(zkVXnP2uawq3HO`AEAWY9p`-x
zk^3?iWE_Lu5K?bo{9DNB#KvEM)mu*wMx}N3v#&v&c2l(j!}+#Y`~QGkts3&Z6Gb}p
z=6<yFtMQW%a}6(tZPE2jNrQQqPRYxQf*hrjO;|kqT}gzux3TAozkXj3zdaU*{s7`a
zqy5pFtS^@P5zJ-LOSAWj%$*c##@OX2XtRqu<;l9Q;1NNAu9Ehe@;>+L{{gw~#Ysdh
z&KBy{p4=M`it=ZzQn#YRtT)rh6ahz8Fz_}}&J5SP-PJU6-Yw82Vx1Zv^UMsL`<SM<
zf!Iul)8)#6YqqX<qycs2Ez5_3D2o~eYoS@7oK^B)nwG@31=95@+^udrD`8Iz2)MC(
zS#LG~=i6r!(rtt&I!}S!qOiLmx6RlXsTZPTF}tIz&Ex7PD{SsFS2JXgc#NJC%!y8B
zGU2tlE+MWA8+!&sEj||H577%r_HEHNH<-)MXqJ)-QmH@AQ&LkeVG}m%d?02XQnZ0M
zcK%;{hK6T4DNJ4fFqd7DQpwxc^}H?E-!kHR;`s%kT?|BiEO*rJSHA)iQ)P}2{}h$`
zHDmC^UqgyNEgkW+!u|^#tN2Q!xr;|x4(;151mOBGjAbvta(j3KY(<Ng8H+{!4uSRl
zAg7|Dom6_2JC6MgfV(2Q5osk$v--;4HZp))Jn701xn7`|E!`4LIUfq5&^?s$j|&ra
zp}W!4`rD$AW&kR6$^JRUEavDqr-~+p>m?gICgxcD7c`G2o+{ek%TokO0zI+z68^rw
zqciqY*fl|B*3})={HH7v&uje26vv}d151|JP044#*#h#w7^Hs47=$qKX{C;*MdaMI
zuF!xn(^BOV`6_@-vkK8%DaFu*mxf?9u1$*#mm%n~S+j3P0q{xAcvf4Mu#-kHLDOto
z+#oD>P_oxL*%DtY2k53r(lNQRMQdq!P_ubjZ*RQ00whJ?5@B@Qyds1vB~@$E`1MK!
zsZPb3uNwQWToA^>V1H9wCWOMVafk(1A?(!Azm45jgLDzl3!(IMcwSQ%>THmAHZh@{
zw>pTYt`$eero+nr+{eqXdteO^mvU6>y%vNSVBcYsbB^I)FmIqaey?C&qd39K!@DF+
zD~|xK&q3;_;@fTlofJC<U4?TmFU+MwomDcfJw}#T=pt1x#~FyeQDC_<qiD=-C_WVA
zjN!LLxhdn(qf2_p{0UFgLd+!8kK-(n?#Yabp4feCp?iAS1|2jW+F9|SMY`m?Z9+*w
zHzM<|4XI(o#IdEqK1?vRWS279SZuF9$R;T)(jC7L?cyT$s8!`nnH|BWzjs>NL-wyj
z*g4}LD{<_Ib%ES9r6=7VUkNa~SO;a;{=Jag_jxW7nfg%tdcBfTR_%*ng8U&ev_Z1F
ztbvRep3GVP+xQF>jHI^um-YF~U4;%d--b}soR*6vHzHhSgb}go#?USYz8Z120TVtF
z<{BjfQ$G8q5sgJxMwHl`Zc4OL%f5lA2p}KyBI6o`UGP>ZH({c%Ifxrn{sLnQNM~Yg
zM~<t4=)fo?wZ)SnoI0Ws<SUzQ2|%6LxKqY2wkn^a9)vexjK#JF@;IvCtthF=V=XI`
zPLJx#(c6@7l1~#F@^eA1ezIX|X2Kb@Etpw^r}k>Ji}s*E`oO!Y(#%+5d%)77x{}&Y
zh%3}7PhyOm(*-$o+;sRM;t4wxOj-?T$Me#TU@)$aeM0L_B^hd=j)6A~AXiM=rj8`Q
zM3KEjWLv8%jc95>wgMT53A;FH1pD#G?u2uj$Pq_NydgwA1K*Sv?LpXmhMSyo_AGJr
z^a(ZIV=oZ1MLWKj_YWloJ8MnOCkP=0bLc@)dHq!Bm@4tozBgd*Gkr46*hGM1C0SlN
zKC}yTQShervM3iDy>HDvZwql(LKYX}_kr}phil{tMCLiua}w46wF%6*_Uf_0cu@fL
z=I$`&Y%YJIl*OK9r#1jmPtzD4mf9hmJ&e$3ZfzjaG+;iNxsy1~G}-OzrKih<xj*n4
z81GuQ#qYa;%tCCj@C(=r?Q+3UTBuh*?!q)Qw*1=C+$5CJqSO(uDr|nS)=1%&<Jd}l
zB^?@zSkoH-y2N_vQ-D8{ZjrZwOY8?|HliREANE7mr-m`Z<1YiyZUy|Qekv-j7+EeW
zZKH(-f%7o#>Po`MX9PKBt!}kW$DU;hFtw2T%bgT<&KPCH;Ub*~r#)(bG<2$Cck*RV
z4oe#OiNY>qa-=zIsL(76oiS0gE6OY)%>Wp<%QQe|hquSraC(UYOU>n&<sd*h5ihtf
zl^k3?7zrmP%z&im+OO`WuruIFH@ShsS)D*<fQC%0dPt7X;O#t|unUZ;?UZ=^2xxZ|
zg#Gp=IO<3+OEJ<3Fv~IGW#R4)9AS7kQi^EthEQkL(bN+M9}Ved8apC3I0i~wW`}!`
z!tS{}_`}2eYL(+aoOyagP4`&$3FCX(s%0Ra5LNF4oQJPH-v9RS3{`)ud=eCs>C{SG
zsh}e=zxTyjCquhbylEG^i*(m!*EC^RoGrlddCZ~`N1OufHs{68<j!-xQ%e+fX9Ka|
z>Cn_g$A&M@AncM;5F2Ow?$2Q4p*t1!b1gALfb)>aZ=508A5GS#KpV^tgt?rE^y0QN
z3)zBD_RnV(5|f;GS)@7GH`v!fISx7-#GRWPIxAh1^J0G~5t~r3W^6AbBoAflDM=S|
zTq9l|r9u3#ju7{!Npjw-CmUn?Mzki}MRB5hUI@HsqFM`|N7&s3F$xx)2+%dX+#}po
zYr;_Ti}NAPVeb%m$lwB}B2~?Q|BZ0HLGZpj*G15-E0c)$zjFhTJ`2TvN=Qz!4|_Rk
ze&8iQE<c)+a9QA|0$q}>j%l&a-yyXIGqMzPzd)$06Se@lUmC43&!td|!nBKd_i_m8
z+{Jd`_$vszv<OB=#R;NaTDFl|{C>U~Bo~B5TJ9Av&A0~8J)n;%IPK1UYG9}{c#nJ5
zwZwD9*y2gt9iIux{HaW;#1hv*Q?#Tcajpnwk62UpV-GZt-l;=Ovhn2gB?0F4DZJ|C
z1UP47$#nyS`7nta9Gu?X2w;|R<BEe|NY@5A(|`;)a7rN1MPd`&AD4*E-LoC53RZ%f
zON=I@+_6f7=8((zOSJ2WUkVhHEAgJ7H2!7pIPa}c_5R)6fhJVipMQB9u=$974&D{G
z_L=z(e~46#+TJS4WkU&IAYKriy9yJ8oJcJ44<Og6JS!Fb0v%5}XzD$G5=4ons2SsB
zQS<@~I^yWN2zyYaiBjuJH7;nNl(%KjMPBr7;(C-wdxH1<yy{&q+!^XQ7HwR%yaz~q
zFiC*k5v6X&jmaPGcen?U+x!m@rUPh@#&aTE-EJh<apZ&0x%n6<&x>?HrJpJ1d<@WS
zA1bH>o@l7a#3iBTt>u;sy_p{-CS_pP7_%zu0;7+bT)SNUNP$ozPsjFa1myztvJk!~
z+I@rD4Q;^RX~aO6r7!k*lo+{9*{>$)$$P@cmo|;XA9EP-YAp6RVMZ|C+BQ+pJ(u+m
z8Ouuoa#wBV(b{H75xy7b;!t@W^m(Fug~W)7cl475;PEOhcoumIKv`5`DFtoWQtLp@
z!2zJk>5IMjnfm7!E2XZ2)CDdQPPt_)cjME9T^=c#v?Q75<YxfQM5J%B0*^}t@eOv4
z@rlCp0P4+Da`3ZIE<A6#)zH0%c5B*~ZgRnJ&vT#@xTiUuRL~zZ58@x_JJBSKOf&xX
z0$~?NH}&4qefA<C`BhL78i@D*4du?$$*^AQWq-Z)T5H9`R|x0EM>R(q%Zac0Z#~}9
ztT1I5>WEGSbIfY1B=vEOAh)pYRM~*n;;j62b8{u`QrLZDNJiT&vEXYUZrr|#BpC|}
z@O)5}Hhr_*KCUG=b4cFec6%L~`e1-IZfv|M%0<DKKgWKw>)!y%%VAU6I7ub{@;5;#
zJ_?2}E9i{)fLAp<ihp_wm;#w(WG(nMfOFPkDKXzW(6G7%_i>Bm@Bk}$@4LoXlVaod
z2)mwm<<?p82qAeQK`zd;Arb8I{{>})Q;QYilJ}i}v>_f;*lc0;rbi@z7DFznbJSuV
z@ITdOIi^VKeB>{u(Lg;4`%7si^v4MT+#L}9U>d3!Z_Q6iECeSQvz;%%v9g*ov2Onq
z(ACiF$24)!XCPD_Ie5INFua`HAIE%7(Cwc5wCP3azYEJPHh{m24@DEbw-Rf8Ns!7Q
zw$bH$RitC$?qDDe5KSh?dE$J9-LlPklZ=EP`U=QH5;-XYh|6fnlAiw>9A@G)q849>
zAPq^vJO7Wc>n3ll=&#ZfR{RFgd@^v#w*bzmg)@#`RQdEfAd1}2Zu9#_be1FDnEwI5
zLkJb&_~Va|o<VSA%#_5@nz2W5k81oQ32~)bY*V-ViEqr__93*$IX4J#8Jc>#n>2+V
z^D~gkDP6IlIBKR*_1aK<78wM^vC}LClpcA~#6x}$2t#BY9Dkh+!kM9YsR!Lg&0Y|>
zt|m^C5I*RvAkJTZP6#;I#GY}-xd^-W@(dIL0j9{I_@B^R+T^lmD1J4!Q^06!LQEA=
zMi>i{+v(4O%{lCk;vtc)Alh@ohsB)p0Aw>m$>I^)W&OmR^-pG4rCYnyDGg*my#uA$
zd5Pmf`8?JfRYBCGjec>j!cLq;p{J&Oj*Io^d^r|psMF>ziRdC&>&x|mNMwM?d!q%)
z-*R}YX6hDX4)x)Kg!d<OvcGRZ(E8>^mq(t=rv3`p9i9akGcVDkjc4~bL*d*Fr2#;d
zHu*JBy{iqyWg^WmM2YB9eUSg}#*ZTd{`H0U--D6sZgrzdRDV-2nUD6iw$@@$m~-OR
zfRTk8p8%o*Op;OfX(Dn-dUY95i+?Q)<OaZzO%?0pgBS4^O<nY;?G`P6*Q#_e-4}qk
zrX<i4BNl_E?5(^jluR)u38*($2d>y^vHRjcuC8=>UT=`nM*UKx2Qv&sqAv0IsY?QC
zpmxTi3OYfZAvBEpmnxqigU1)J=FF!B=5?kn*I;xlyL5?*3_K6v*96i&Si;90%Q)P`
z9I(`~g!8mF*wP}4m-t>-9%q^e+GDTfK-^wz-PyVgTONSYws*&<6-sQo&W;C#Ftei}
zh9cRx6`?Z!q=kH{Xs2Zd3m0a96GYpf6m>2W;EpKGy#D4PIb}b1Rj0~AGpQr0t3lDs
z9F#}I#UgTN<eEJ05bZ3IAT<_Uy%9kwB&&$d-H|zKLy;6X4W~Frl&jCNnVnhuxF!h9
zlU4KWYY}!{t-54?NVFRZ(<heU_IPG^iNamyxY%h#iNgHHe9v>Pg3NXr7~v>${JSFy
z2FWH2;lXFZ+*R=Aq74fDCq@;pw`o#VFK83!Qcvz35*z!3cz-cw?bvHHR2JW}yY8=Y
z_mu=;3ydirqKP2Q<Bf#4O4Syw7LbL_G8QOz0HjZxa|O5pD5-L%$KmEUVP4gkkk~*o
zE2&~@j4L1aRi&-jA;2H&R+f3DE&7GI{Je^W!eKnv_zW#^Y(>5h;8x@5lvrhaAyL4I
zHAFg54%HnD93Gik8%zzyGCWG*)MGG(t%-1b*y6D(cufGz!i0yJa<1P)xskccpB8hA
zb~9uhV-g=Oki>f_>z)cT9$K4Y<8=tSBAj7*SY-aRZuy$=KF~YBTU~Lsz#L8PDQfYz
z^*|_~?rmf5^~*PvZtajz0FT=?03=6=Sq|8cu#1Sqn?5Sq>9O`}=H6jrklaLh`xqw(
zbJH?YGhVClfuQ_hrBjvpy73{Ll$JTqqisekvgUp}j!@W%=rJ(0-YxlW*5z1cbN)7O
z`v<jsJ5YezjX5^n66xVBl@{#l)Tw9R0@TF->t$qh2_Z`)6>+V?&eCEPq)HQkoN6~m
ztfxf!d!!Bc(2FIv1aUc#cq5I!u>sVk$c|*jRwYJ1Lh_;br~&BFx@w%V4KbXrd>v_6
zUAb+^h?B64^=8`+go5H!A$k?gOy?>#ZG0~i=H6pNGjVLb9RTZtrDqTY7D@Zj`xIX9
znS%%kPZZ!`jNE>a9ieiqIYX>2vc5mTgJTvwJS*5aqSnEtvll(QcLh?+L7MpBod`P@
zH2M*TpC`cGslUpBuy1E*=ZcQsN4r4f{@;W9n$#uM+ZE7nOeDcV{kXvr^!wfT%axOQ
zA$kbb*Vsh3o1VG0Zsh%i&^#Sbm=5IpvD)rnE@0wvtg}Y}xEyAUeRfX(H@L4@<HZJ&
zE*@Lrn!SkQTJhQsqx=uhuwXR$M3etm5=5lMN#+3obt7nQEVEn$Uu^n=fhk9&(YH5*
zS%G0AyAvIyPZj89kjfoW%Opv=U$85IPkSEReM^9A$^*PT$u8~CeZcB-4I-}vn;JCC
zi>E|8!=ZFmlB@1l8<9Au48*d{B@(tUHE##6Z2@Fbn!=l)Rf4%&qI{C3>oZ%y{=XjB
z67LCjpZ9f6OZ-QWdmWbzv0NKuZX8~X8Hfi3z*nhJ|4(6Om2Rih*-vjT7;inJ!p=5!
z?@3@zb27)plSI11F}Hsx3Z^7Yd1i;zJE7g!cobBUj|(BbJdw1=kD_RzF5N1w*5$uY
z5MeIAK$Q8*YCIwayP-?hXK(e0fYP}K*?_VhT-?)$g%FfG(Ov+UIXE@u=_6R5z}O<R
zB<J~~g=%==8}II#!xcoz3l0!=ejE*%E;juIz&Q@TuY)B$_750NHmQP#O$JTt71Du)
zr$FQ>w^!;CNCyPfi(MJyMFWP-rUK1;`Cvf=SB))%nRi%~qp}_gO$DK-x<G$kVSmx>
ztJRj6dm5N?YfpC`wK!god$=Ck?&dMw)bXbS*IAJq^y&a}$Nr#hT3JiqaDbCTp$*j!
zj;}4T?14be6&Y(64t8sCi7<a97l-$Wbg`5FWOTO<Djy*Ml0{l)JS)rv(Df`%0@-=Q
zJ3=*LSqg9z58-s>MW_7`NT-Y5rxcKnXdr%>G?qTJeA0%|&0Y->F--hXBwbIB^9;n2
zhe4TH+%Bftts<!2G!!9$ZygTkd^z(>X0`dtk>yJqLMJ7fa|D^qy*xhNkBc+O1x6hO
z>^{U9n<9@n8lZIQv2mVg*AYi4(SKY?ramicV}GPzVrdmZ{CtW?r`j^a?mJqK2XLwb
zGTPqt1Zet+9=2#xxIV1ekVx<Ecwu5hx=LB*L@1cfs}Q`^Ai_y?Dz7(Bf_Cj$k$7(+
z&N&&x39=jY<^Q{<l&=sx^vBw#mN+ykr)B#6X2K{G8<RL(VYj{xxl&_h^;zXKP=;y?
zY8<CKvIfiug<S*`7v*=fSBM!P-8)uq0d7m2oM4O89p^W`N{_>!9@~h-5`hbk{-~V6
zcP@ve8EG)i|L2lC9%tCme9{2;66gB=IFlGI5>icx)t?dK5^!4~%bR##kQ0|qpIk-K
zHnW{oKIPmYor{%)z!*fsamLw%J)4dhA3yvB%2UbsSp8f_)6QkY`Tk={6;-r1{tDp+
zk)N?70zTmaFw((1tWu53MY(W#i$=%R^DYE(J4>Pxm;MbpGr7_%KPW3+4B};G5Q~{+
z)L?4z-}=KP;O??aCUM+lkZudzIo9Hy%K@AjO9n2%I-~CjAQ~1&k*6t`MmYU;^Q$1t
zq2%*YYHCc7SA)V2wncG<!kMiW9>h7<0GUPmRpJ^^E;lbZv1f@ph4_;+5LpxsxDLog
z?qm`E$Muk|J2#G<@t$ZiP|~TYF4KM#=4wgy#VeLK061B_d#jm#H9<~?DVR6mMLQjy
z^W=EKmESTqmUOrhXE%3<0H;$)YUh>sNKh^h+l{#W=8}%zl%+hszXi-w6K4^5H>LnX
z0-YAyvh?og2@MFH!em;o{jJ0@i+CPnpuHA92!X?TIO2rc9ByOG?0S1ikBinGnPhk0
z0hH&OBz(M30N--Li1q$Kn8r-~WxhMh$JA7M>d`ES|I>2Tl?mi?W8zQ|q*5E|j%O8i
z`E>n954hlNkle+Z@p5wyr0cF_h%0cI{j|XPxfoqqS!*XR2z%XIV#`yoyk=D60D-yn
zvSaGS?kn+<+N3siaoq!eZVH^oHb?Ix&~!E(ZMx%3Q63VAm6IK7()4*)xEsE?H+hn5
z;|8i=^3s7KIohoa49y&>!Lj7vM~Ua!qb@hZqH&@SYVVnmhcHV7rtemU&5#fm5=P_J
zR<zql*Gu@gZvcpW7?X27P7Dt;rtl74q?H_MiGMnsR^(EW?7uHM4`ogY{qgT70L(SC
z7v<DBW`7chCS^ja#8FR||2YmOmqkAaa?_#N5zjsY>DuW9c=jznH;_s)bBR5kC623u
zm*|1GQFNZq^rVljAfFXRw$jmyUq1&y9?9U22}h(`CB1%3gQwf}4~4tiFgmac(WTFb
ze*xx$7t>X;8R8f<r2*4<nPT<yHu_M{6Ps?z<2~`z3xr+FBzZXKMQB$|LKC){PYI!r
z+8M=W|AxrCV<*vzFs4nAzretKG6Jgpmq1(^_Nqyqu5H7af*sM<xpAG4JVm7klaPqc
z^9mS!({GNA6y>C7TedD9CQk@<d$i!R;k4HwsSry~9Q6iamyleN)!Y4ooK4bn)g-ge
zn_#(jv0bf3>stV31WwPZagu1uwUIg^<?Kh(AYJwB@izZB^Rb-V)_Dg~ji^T`re#8c
zq_`^+@-z|WzDu-Rdu}n9XV&-+h>O6)!Y*ar{{pxOoDD|DrJ}tij2#zOzYpaO(Su==
zt|8$n`}GgNseC&}p*s|Ip&MGUdT24``jCHI=<eR8Xcj4sv7+CqusN2-FH_ZH4Tz`S
zlE0ewKPHw-iw6)6IUfm766`@&{gkj%Z^rk=CL&!|7_QatZ6R*Q>QqG4`Q~D=&wyQH
zwy%A0+vm`vi3<~63w5}Yu5*#XE~4J3lQ3<gFM&J`IB<7xlgJ?J6PS6#b!mIuDDMQ}
zo+&slXnQAn#uL5*B{v)b#F}3dc1chWZHhfay9ykFNGqlZaW9f%d+w=ugyjDUrY#$G
z6xFyvSiPZ>GUP2n%$-iQ{8PVy)a)w*z<5Ixg_tJgtZxasnzD(K75vQK0l8ds!uHs^
zL8GM0n{%SghPo+UOI#w546zND)$FkEAzgCC!B07-mmk2~oXo!%r(P$3a>$<PA%$H9
zEE4dx6i@#MlDiLEulaw5%)CH10-fV(EH8`%^a|Hhg`K)?IeCdfK<U~e<89G-l;UMZ
zmQJ(HG&;{S5RZy*wPcE7cLWE_0_b9?NvOppvx1PP7MR@?&Sm1LpIW$In7?3`uXk{6
z6T)bn)XNJE_lxiV8#5smoeio!ZAp&NA*0-gx%>rdWlyXiLg((uv4Mj14#;etHcvax
z4d{ANK2EnS(Jm<WtO~EhjGhNNZ=e~!NjG8uGaFEIjNK8}B1Z_wLkdaWHzHjV>_ntb
zH}AaV3r;-pMkxShXg}UGA7K}c1vm5WEP*bGTu#OfqMbzt4_SO5vc7Lkol}~ro6Qdj
z!{FaQ^oVc@lco6bq-Yq(Eg96C3qZLJ2+io6b1&$>xg;4oHhvIA`FO%19{3euXRYhd
z%0iHNdf?JAy%xmL>3hNCoxG;Z{Tm2#K(a7e|2rZ4t*va}maJYh`bKBmEV{mv(xo<o
zJ{gwwEX*Hy#Q6L|Bb_YJoku32=?>sjVXiE~R$iW71lol}OD^qJbuB+#u-R7k_yOCl
z6lz|uCt}xl@1g)=m9{0Z)nZWa0z0?3RAJ2rGK7l-mw?I*%}c=axt1kCT;OhWIHx5k
z?&$)}K-q>axfEnQJA5+4ga#l*%JvwsborngDadxqK{^|zb9L=GmM=+g*knV}DZqW%
z>V&m;Nsw8>5Ua#GE0k}Tuu$>8Q2_PQ#%u2t37au3JP=%0yI%>E+E%!lY>Pug5x=9e
zH=0)_>^9&qK)0m_yd%tIU>~8^ed7J!0nzGo`QG^Tst`I{_r)g)lCT%)%U6SPNjc9C
z#oVhyyYXzxZxQfjg1unKssovjFO(M=2Chshh^G%&(*!~ZHcTSSxOS{=P|00%EfC7k
z+!q@uNOkZm6;l*;(I?9wh%N59f^rwu?h4aLA?m`s`W!Dc{+5*@e?K$4q`{M9yu5=3
z;Rql%{3NVsZWQT;AE->FJ3cAEc{FL-?2p%kxE<I%!S=L^`FK=8le}DB!`RG?0i`^+
zVeF4{$3jrC-l14}9KqZ+Xu>5i&<tU&V0CIcVnSZ#7Nl;?O0~jx!kJO5It+o6gi!Qq
z4>EJ!uA5LI;LwwuU2H6*-n|$llSJ2hAc`zh=2$_lFHbbHh_APHiPMKkp{%+7Eyy`D
z6y#*)-E}~m@Q~E@w_3M+ge-(DKBiUc15tC<@k;b&0-;`4TrR>{avw)APuc*$43+0D
zcDS<VJ-dNoNX(UCV$v-_J$SKOP<xQq6Ah446#j~2YW1x^XQOAr@iVy*fNPHqU7x0c
z0|mifR7K-fg<U6|iQ_iW{+>ryFaY^7M`qI;XKc)$W{G_Y#pOcWqPkYY;qXA5x=D$J
zG=P;mUK8Tp*3~f>%WMh>lPcY)$Q>lYX(op(+{<oO5Ke}Y$xYn6eBa7_(K{k@F)$Wp
zSM^yVLMOL%FKs~tvo^cCl<VNPCW4Z!&W^GDmJskjXU0)mmGn}Eypy#xh?yn@AZ>Mi
zx2-=wAuXQRzWj|JYEF&a@tYlhG$GLghhjBRF5zUhXS8;^klYF7R-5w75ahL`*b^mL
z-!VJ#hx$LP>)45~tAY@N=haw2u`>`#41^f5&d!8$ch`BgDjH_zjs1qZK<0%?Ze7#G
z<evpn+P6-h!AAQbQO={WB*i~>bvneUp(XmfNVvytuo|=O=I^EThD1|T?gS}Uy8CUy
z+>bgrE%wA7y8}2ARKv1^*7)xpB^vK9ae&%)PXKd{;jL5kg+l71j>&%D574fV9z)in
zRkb)>pxN8XETHS7=LEqcY4^|aN5b{?;1YqyqlCE3cv+Y8q-tEhHyATjWomq+AhnfT
zGS=A#!a2%$RWqg``0EqsVzF`Ri{nJ|MU4iES4GrY+fDZ1zQ2eMKQ3x2G>@2}6(QQ6
zAg1n+&fwC`KrS`Xb-HbQA%qgLaXYGopc$+~T6c`E0=P!H%SdcFP#ArILdnI!tgR+#
z(jIiY|0<Fq!;ngB)n<ZYKW<Y*ctLA!mmgK;7WuNdydB&nLS(BG;a(je<_#h+EcUe7
zU1AcT%O>Fhq7DJB9P<}iAiR~^31t2u3T=tgx}e=ZlKvbs2`l$Z-6bVt4@h383&_#f
z>0mNQkBg&)=JFyYNe)%jNC}5u7~Y<vD9(?^2AK^+=OXDrZw^uG^#Cy{{OI<7MEQem
zyyV1=z0kSty6eC+Rfwy}qtOU4UfZt#5_DH%f&TK%fR>e7EGNW+-X>1)K27oaOy&<a
zACi3bt~7UtusTl=%3FvxqY)EzZjG8+oGHwOMf%hk7fyku^K<NpHx>5mt9P>A%YStY
zOpA{db`7|ZMic0(2A~T~-aMOH62{PdC^|)(3kWAVu;f^4T1g(?gfUU1r<QRe<6Tjd
zL(UMUOegHjtN?bz{s3+Qy)2~rjQa(-JF($Hu%=c#<p4mEWOb*AXNscNCg1swi!g`k
zLLw&HI}QYO!i@g3?|(^<6K>U^jH3nD?FW^w(*pr+i8T%`U*k?8T^StRfLPfj!j0Kx
z5Gzk0>FowO#5Ka4FBh)3?&d^x2oPl-pG<V`5#{1aqQ#pfX(~As)QoA805%S90Eh^t
zp}lfgiGgz1_+hco;Q($1sc@hd`h^fU-OoO2yCVp@Y*_uFh)UUB5au4D8=iRLNa)<R
zB$8%9hyS}ARlbxHrQ}cLXJMp_v?x|SnsDw*bgfpAE=H{~S#`Neh_h+oC4kuLPyPVa
zFa3SZvE_S2D6z(IkgmB}x)zr;06k*dj2#*=T$OIRh@Q8L1&#-+k4$FwJw&^0(THGT
z;)E8@3d}`A5Y1dN;{=dA;yG8UI2W7<<kXo%r59F><4*;1>KrNu;u6u0!)di6J)F1v
z8DQ?@_!C+)Q#cuC-FZ=zGuDL(3;kC@a>i0<r{}G57LYT+jJZGFKfBO~EGx10U!dva
zsG0Cyg9xX{ZbWQ4?i>)GLpYKaCHWQ41<o0<h-jyKg&-Pl8WPpX=QYxyaXDQq_E!)W
zqgBr|JSo}<U|%9RX}r^j!ug#0)W7)?3&v5{6io)Wn~Pg6DxbF|J&9UucrlR6p1phR
zFNCT|$so45r2O5dqu7+#U5JZN)eK3)-SzhZA*4(GwH^`VWYL}ICeneImPjnpGUbmu
z1yMAVg<_k_2)h-f2#h9hEt)R}qa;%0kJGP&aL<;)*Y#IH<_79Owe4Qf&afL%CKEu<
z)dfM*T;j={t^si#m~1B#xVS@@zr$m-<R!;m3qr08y4H9n6I}d^irKH@ACCq``M6>6
z==A_z*6Ooc64pL1+&t>(^&46j-T>s`jAzy4?29tKBQ(z}pzU0_0f;4dpkMhW!me=(
zmtFgc%r#{5K-0f=Gl;X1zLN&b#9RD*4~7O`DeMo>*3+6g&#nGf2HhJg?5ta*@Ef1y
zKe?ZarSEV=rid7m!;*scD$IIaZHi|Vtk3LZ=ec>we}KA3T<2i4Rf|Uixfty{JwiJ?
zB?J!Z^bw2S3E_t1e1dt5XgJW1e%-8hJ0jd487qq551GpyuCUWgx(Vot2+1>5I{D6g
z4?v#EeF3c7i5Ws2t+z?7z3jaJ&K|WBURe<BnFCv$i$%E!`q0Uk<NgvE=L*%>MRevN
z3U33^)rf#>e^eYTn$&dA=g=-nBa7yXH6MU5r_hG$kap6wf}9+ySaWae_F##FH4OWU
zTKrd#%ZtawuJ~LuC2YqikX2-rhar^(va?uA1i2$rV5a^|6#Q%Fy{|<cA)L7-U%GL&
z08i2Rluu$$JnHYqrnT`cQ64=wudq7a!H<ENKO=^Zh<}Q7vtY*|Kb7l04&sbzC?<8p
zJ_4LktunQjSJD{U6@lgj%AIM=JxUN|mm(prP>FKhtjp5vlgGM(`DJ#ng`NQOyxXRE
zsK?vvT?AA0DLkxp*^?y-Q}cix71sgfdBNo80&VgXRL*Y*pXt4bWVR4SjJHmo(T0<2
zQ7}4b<Inaqg!@`A_F7p4dDt_ct`)7*CD*rqdJf3t<930oHJ5Lte*sfI^qb<9=LzS<
zvKLiW)-*hQ2s9g67*GSa_5}d4XJm|v_eJ<VNb~P6F5`M%<R8XsUu!J-GC_A7-YM#i
z;iA3KP4}ZRy-64I=jsoaQ8O`mobopw&*kqqLZlajM!!Ndvxxg7J>kfB5SGi%OVpF;
z)jUNdFgGr0p7H~`-m4`Gj-DuzU`)lC?-If8K3F)(QRN*%{4Hw@3nmW{yaq%wRJe2M
zif2V-#w3UHqhAMbk8Ed14Sxd?&dSH*8*e&{KZ0m_i?FleajY6f7&i+cW(BLH+Z4{j
zzP%;(cpK8yvZr-)tKt}exn~ct?TAYSI7#e6cqS+Q`!0yPg|)QOUUl?)B_3?!%n(!m
z1CmFUZtk@m{F^W*kEa6iyH9{wftd+A7dGR9V1=1_?EB>lBvmZ0*EWC=6Go327Poys
z4CkZgG*fA|`w+}^Q;%d}Vig(j5n$p)U(EM0VRwflK_q2Oq)P~P`M5tu)xE0$;NXF!
z>=^-RgdoECUPNQ1)U{ViX6a8Hi7A_VHW6wp*7t`Mc3Baja^U9w?}eG4NLn~1B(K}c
zed@2I^02kSZUYQ$qyw_aX9a;Pl7{Rd#Er|86eo!^Us<l&WCOd*=RmGoI!(ls1|Y^`
z<uAa42?+<PqhCN%axUIt%9n&)h8kychQ-lB92<!!o)blz=Q28`b+t@>RU)+y4Di6o
z|3QlxtOJ{VOV~wjVq<W-$U5)4*eH$ruKX`QG;51=3FLAXA>sOs2yJMsy!|~9T<4Y{
zHa&3e9YIbLaW{;6_ZM+`2f0W^AO8p9(4s?3z_GIkrj`M86BYE#$P=P1qMSI~ONtjC
z<qw&yMo0C>^1tk-q|kMfAZO0TYV5FhtN}FZd7F6RCt|ot90F?i=UC={AY_V)Gs=fY
z{tN}TdSTCJ3cKsHN^ey?deTf|>V?Aoirq?A>^n0MwL#3yoAV-^ev9TJe0R<Qq7+%Q
z{xVznnga|g)<9e*s6LY;LzP(zyWEEb=NWLYi!HRYfmfa#(CyEYOC52gXp&}njbG1E
zK9ITu$6-3)U2_7Oh3>S7Oy??Tj2J&|eDux@P3h$}28+Rxhz}cL>TLpDB^Ftkp<O!<
zNSTkMQ=(aSf%%*gjWZ%X6j6p)^18ra|9bv{BA;$c&sFuu@C87ffplOQ7LN#_pGq?r
z^1KBhU5#3mgCR9*{uNMOg{o7B;z-f3LK~FR8vn5~9E=|tbayY^`jmx;>-_NXh^|H)
z{u>ZYF=&Ry_}@CIZmuNPQP^F8nL-B}loO(0I6`E%Oeeo%;etuVpJwCni-9>Cyv|7U
z^{fy#subPnFkJn8u{fantQ2CZF=B}ltJ1<F+R3bP%OwHL0_LYAnCued^n1|nfW2P|
zaZ+p(=zF^^1&|v`W}|35Fo@S(x<uo)gZ*T;05Pw9FglkZoL6lft`J-4Xtw#X<(p~}
zlOHL-Q;9BreiBK0u$?1gMD5E#D@oSK+VbVw0UT@%#MuH|ndXju?Bi<jvmi5?RX}P7
z_&;8ue8-lR^YQ)y@>Zk%P@c*cAL<Wso5VKZofQewmTAWPT!eYV=2(e-zOtjDMi6uV
zj<DHNlW348fs50Hk^uK*)%dH3-1V41RMeMN17gBph@r~3?XZ%@Bz7qr`t=t3&1)2t
z>t(%2qLd#MYMyCneqSUr-?a8vdrg9xg!D3MY$3}1M203_;cgOGZ(Mms+Gi~Q%E|V0
zD4IpUzZUHT^S<#=VQxxk>LIdar-RqI5x_3R*wMq{VUZ*wCAnxH4dG1Wjd3VW72x_)
zcLpsYctI|m7IO7to|X|<7_FIb^D!W9(R6CT^fKNSrnb@b(XL}5T#hMKj-?{q7(9Qf
z4TLhAF4U<o*8E7`DdWqB(s{Xk0;IF%epS2Nm)8c#ljjf**d<Z-Z`K9OlczQTGp!E|
z%h^0LuWusCg=xn@!*r2OL2J2Fcw_@WmjxYTMiV`+x*?ePgKbxGKDGHq1>%fB;<eaD
zkn`sD4i9G=LzA1v7bE_RO-#PUiGXQ-rA;B-2vZPApuDs>fJ@U;oub!9xs=?!0VZ)(
z>S6Z^G!q#zgR;lK!sWh3R8Cx20Z6F*+bv3_BXA8nUZg96NFb?4&oi+k#@2~z|1{Qi
z*%H+3#{Hp8<aXN%#C+kni1#tN`utm$ScnsOD<_r_lKL5o?yB2BFklg}#8C>nx>i-W
zQ-HaZrp=|cbp%~JvfXTnkpdinNx9v$Q$QC1hCZkWZcVh&V!7=g%dLUm(9!^$a|dur
zA;H|^3OfzNz9?qRyAuGTa|(9G3X)t_Lz^g)&&DChctK%*hR+`cOzaFmH8H^(iUUNL
zsWKCgeb2(X0GWNYYBIFCcvq0jzU-<kz7R&|q(`hUGFI3P%JpT<9uphwUcTU|E_=|u
zSC|Welo+|pzIy;TBm7RV6^lM0&M2!G)Z%bq^-)*j?3T`7j|!#2bY7Ved@9n_(5X;U
z_)2>MQOy>#+hZ>g{$FoRNGtZ12E;j=oA>_Majz1q3CW$N(k4N!*c4e!Rz<towa%E{
zpvI1mpBkS_cCJDDKS6G0y#l+$AN;kdfXbRE(&n+~v|P^hN9fFWPNJv`-XX;NkQdD~
z^-kEk#Nx23cL;if(ApTm#T@$(&YdJ(1br@m0<$fmAKt%j`8XN))MCvFNM?B#Q+YR%
zp&c8Td<Syy$81fQn<2kcq56H$1ey;!SUqR=1!(9sgPY&UCV>RzYXLGVxOI)0tB|fN
zErv~XtSrPummx-594sJjJpH5#*Mle$3wA;@T>LE3EmZ4bfw-g%+6nXEdna7nVp56G
zJjnKFl}>0E4*T_VCv%!0SUA9*b;qvqH|ew2t;K5H{+neoR_Y<_>?d)bcwH}~`9i6E
zdC`KmLr`~<Y1|2laz%W$@7GIe@q8l^ujj7SPbAl)dx)mGT(lcT$3kpuhvF(>^yzjS
z;lyAK!WCuul0VWB1LYI6G6vHVeh)V=&Go6Hy)4X}*7kB*JTVAB^;@t+dRgH-K{Hg^
za6iGMG1KINqO-&)E@`~nB-BNa8Y)ZXzl6BdoDdP*)6YAmly9><TuaI^LUTKDalnW@
zzk#qwcEf{fh2&w4!T~3&=$H-!3z8<#eEXOG`6Ag{FC)kWpcC;hHyvqzVSGaYt8tVF
z=S#<9Sj3zMfWQv+SCzz{<qs-pXj0^}T0AGr%|4!4b)gxM<`R!#vZqgb(7S~?b7T(v
zNlrcf5HQyvJyyes>ODa&0vGPmseJq}5Vwf-E7cfwc=^gAzgk=^D6g^dU7B*7^@#Fe
z4dE7UrUg(}7T1orzws}pP2?hbA6cTK<jU^gKmqFbNSF>+*u}!9pCn-)3UQVU1^YsY
zw~z8CgS=Kb&(Va<73oK=AkuvpMQo0aqA3iT#F$ky;&xKdtNz4)t}>S09PQ#5fZQPl
zd$3uIsX`nDZ;_Pt5&`O`%%!7_Bb-Mu3W3ttI7bi*VlUpe8voLS@D$rGLAh}lhG{oa
zJs!*zZ0hJ`Z=jb=zc`_!+*O&9jKsD+5r~Y3I7m!Vn6b@t%XDynD3`js17Ag5@v#th
z4V(a|18P;Bd=e<EkcXdmT7=owiNVVKCqueqY-w9Nc%>s=IVH!JF4wlFL8L6Dc(A)D
z3e(3VFzIx{RJpS=9#$|jR96MD;GY5VK}nC{u*uXJeWYOLhSUPJJxaHWFf&uW|LD$f
z;h6<W4}&w1r=3;4<N~gf&CXEVEsO#T^~c8wQpFJy#!iS|pIzem2+M~1Sz+dk*g&Uf
zW@No2)SZUC!-&}VFTbQG#JQryInFJy(>V}uu2pth?XkPa^6l6-Rdnt{Qmh|{M}@eB
zD=o=)G}S%f+!9S^273A;=YizTAxX40U#ALl;`l&geiH4ZywTI#a+N?grDUtOh;$>g
zrfZ6!^8w7+3fJE{Wc^8y^U@SmlY^h9gt_3{)kx*$_zOx(xJ&08ccuWEy@HMO;Dv-;
z6f_-pHc%(r&jjX%<<2JQzwG<Bk_w}YzV<f(t`v(#3l@bX0*7g8{hk09I9;5wNt}2w
z5Zy0ngT(ft9D$Qx^6GY$AlF*k@b;MH?*+hv6D#PX2EZ*Lmtn67AWeED%FbV029?_l
z^Ld8)7MFu~XHgtVr9;sb;Pi%GmW=o@6MBs^MqkN4?(RDEF_;s#Cko~Rq#j+-A;NsC
z^&)IY{<m(u8qnR+jyuy^2s>T_YHqSlSQxtTwFOi1RT)}dC(K_<cL$DWExSdiE5tKQ
z>L>%(fzW)-JeB&I!g(Ozzq<$BYsT`_>p{)MYF85%&{y6FU{0grEW?wS_oiP!c-??g
zw+OP#fr8@Io24Pu{C79=hr1?Q@Yqfy9B0RlzQgyTTtF^0q=$UTEsl;tH!`kR=2l1s
zb_=JO+X$K!x@%{X)Dy1@%RP{JMKTpM!EOh17Mz!l6QvHzboRc&v;wN7)wt^ph+KVN
zkZV{>`Uj{>gyO5D497GOE!|}BKm%#j+oS#Qr4Ta>FQ6z#O}MipK6w!L&%9K?S@h3B
z>GpC{B^OwCiKaX##&pC?cbU+t7f;^};mW2-N-FUm3!^5?3-Q0k|4HB`vzZO=;s0E8
z?va?(_7vh_#__!~*1WgGK&mQ^ye=fiKqoHVZO~I#hvL7DFPITfRErJn;|o`{saLAe
z+X?U(N&A%kd^O!GoXR108Hkfa<O#?|-tmec7njqzR;L~BF9=sz@|(8!10YVDmwR-&
zw%BaCaMEUyA0Kasa6&TTg|A0H2$H8m)O1f4nFkR^<0Q!Vr!ap+H8eC{6k=A?bTC}<
zA!zq4>kn#P+C`v0lP4CUwFx0F+Ecfr2a2R*O-k}I5xEtc5v(y-;|^i&o!u?*u_#X$
zRBrMR5*}UHJ@+uM%ff1bbL~e8J!1Um;c<XyHv(1Wbn(ff0C~P-J(E0o=Es0s)VAJU
zuJ>YYA?iS!uJ%yaIWQ%+#Yv(mH+E+o@qXh^oPc)5evkW438Uj25u}2eRdoH6aGo3J
zP})_TCn)!ier_(}`X~G`vm>u}h~Q&A#<|XugmbeX-b|~<8^UrwVFw^N@()h|F$MJW
z#V$`1bd@A3huv#D1LA7nJgG0X67Ad(?Xnzo3UF7zm^BU)nG1}ESos_~S&*9`yMRv;
zwN0J{rlj(_w7tUqN}l%BwD125$Qi*hGWu8m3~bjzdinDZd2X4;D<qYUNpNE37aWD=
zVWJV?y68y6{o&6-+%xtYjDuf<B7Y7vG4H?q;gE7ZR|J1(lX%ff4iE4c%dcK0?EY#e
z28z6fK$l#eEVMs;R1n`|YZs3xoY{r-Y&BjN;5N~%14*El{c1^rhlbPj&f|h8Y7I@6
zo>=m=@=?0WQpVGT(Q4gY@$ef2b5~>k9>={|=t+`hB-)(7n^-?j4=nIjiGl7>E%p*^
z&h$>>5j7|{&1%>mtB7#>HgRt3ZIB#lV#9a&*VREiCfzi}mcraNMIeX(RVaz_AUIx>
z8lPeGpKg_6{`ZJLg+{Cut0)KuQ9Ot-3O5!}tUzTxaYh5}=;1zD7eBRF>3!n5qVg5V
z`t`C9x0$rAtu6fE2PIlEKJ1SjJ_Iobaq>Mr+C;nAIPP<M`<{T(%pBdi;;fH<`OVna
z`V(k(b4sHHG#O27{wXl^PM%&)7vXfZPe*bjz$Mn(9n@j;XCTxad#N}|Ve=lV%D7u(
zy+^0F&>#66#ChtvM^#p;{a*mPRgx<<3C36a63kii6cBx24FQhCF54ccpA_aCxbc@i
zjB#IqP<*vyd@I6fp!mRCwEWiqv{`RQY_Fg>#I_lm9?psX2jr$>%c99fvV%>8=7MlP
z)fqp213-=2InYh}mN3~>d$6o}Uqoq@aq+QedTFvGjOpJ&<*iAT;|-#-ftdMwKsc5r
z?OH4?Qpss9-Bd)q4K0g1L=ah{uMubd;P5!!U0C=>!e%^n?Xi+bXT@-=PUHMniG2mS
z9-O+l!02y$#O5sa{K@I{p|~{n{|Gy4IWgB;H@_3)&w2fWb?eTV#?~{z9e=tb8;Db8
z2FycS4x4A21)2hDqp*X*b@q%K86VAB=t&iv+x;looujXp{dw9hqVYPLli>Nict>Hk
zcY4YSpNg}WL<U$;Vud-N%?TtVv5iRQ$?l~Zi_BRPkV^sPi^b+DF}PapjA3(^1X|g~
z?j+JVaw^h0lGh2*0MV)Sj(N)0EayDqHZL?yp4@H6?V^ZE%t|~X!YOe>rsa3t`9R#G
z*|S;2eivctGpHW5#Ggf}Kh&BBW7GK|)c6!@dxbNfyD*NMz5uj&*P|T+N<tb~lNJPa
z4hZKZl<tVde+8C#IGE&B>k1)ZTw28wzb5RKN3)ta0h6tT{5J|6v6aGRFKP>MtjIiw
zXm06RO#Us9v#btbxhD^3%Lf?M7&&~z_*i>k2y&Bx++P%?wz|J<iG3FFKa<&+j$V|o
zOR1HeDlfQLL9i;OEn)*9W&#_Lp?FDjp3yKJ*W{@k{5p#R*T+#eny3S@fgq<YZLyx-
z=n~@Y&0Q;-=bIWpe{XtBVu?l!R!%IWmUJRDoT1r+?X;ypm|hU=pRb@Z<v$d&@jI|I
z5WGWX)r?qlevvRG`GfmZ1Wf5;M*ZWmgk3mv#<fCUu^b2$<+?h_?|&3s&Q{z$a85&X
zkC(CK`NO%k<+=58L7oew<i#l#x#@B%0J^rx*;>3IAm>T>7-~N@q9i+pxMd}xP{eMm
zf1Xs>`5+q7y+ORRa!Hb<f~R#3_+0@eAp&fPiv+kNh(&oJOy(04R{<ndrtf%HVMl1k
zmP@x0czj&d5#+(OZ8gH4pkaF5@8JVNb2q?tCLV3XlBKX-$RPLaS1*anQ(g^u7Q6Rh
zfO&i%5Wwi6Ck_+ly6Ngk!{~`MfheRljJvK?KEY29De?bv!u(0vL9lA9GaSeVm9%Et
z@TVmz!W~tI2wl~#GXluvXHjg48KUVJsCW*<xgy+Bu$ftO*hl~|tfMK`Q#em{Z2qR_
zzp&bmc}DS9W|c%28VT5)3iSxl3Wud{8X-3bCe1+{b>7^FqMOj{qB6#eCW_l8?H1C`
zaZ`aVIdafS>?GQ~i1Td|8>cA^q>n)Z%P$n7DNU!8RmVV>|5|6%En<hUCDE=SM)|#>
zohZ@`?CD~aaUiZX!p_)zJR~V%pfC`Nt_@MIA$kYMA~c6B^m|~Jl&67OqFJ<)Kn@u5
zuLJ4Yab#u9)Hf%r`-^YbJe($)rqiJ(Yv4R5$jNAGeLcrOmVupQ654+v$Tg(&T%U1O
z)V)3!>}Ee5`zcJ}(fw?Xjt%^WB0ZGDQgfcZ5qRF1)UD0X;ucCWx^&%jW5RjXoytb8
zgA0V+Hz^ou0ad2hTI?&V-YwY{vSm&?(b{I<OvEsQ)UMeA%GK%`Y>wq7LOOdALLn-3
znhk_HOM9nBLuQ&V_wJ-#%W3U8L8MI6F_XQQDLAy^nAmK~l5h{os@Wg@eL~&kxyoW*
ze^h`AEn%s}IyBJCTZ6l;>!zO?jaLgMNAf^V>swK7TA7`*lrXuiyDea@Aj{A{L~6th
z;GtvI?I7G$q!1QMigp5scG&4i4*sz~X8Pu7vGn!?-9OQ^kh7m%b^xJano$edM`5Ri
z-D5wFU7^i2bf5K5L`&R~<Eslips?$NdNL!BzKE=IsU3mM6WrvmJs!VPNtY{~YD^Vv
z&f^!EDZDH0YCxRm(~IJ3?o2EVbUM|2yOb1~CEgg^725TXZ+JS({X$%GX>idqnAn!v
z9WYN|*b?Z>eR2arIXd;Ty9BwsdM0QnuGymiST*y}K6?Vd^T}$mw?#N7+)JU&E1Abl
zdjaMSW)Vhz-0%l5=ibVSpl`SQV}ZDHPZxk!3v%vBX(JZc8z9diavX(@)IK1&9fo@F
zPffRP5|&$@3)py1bgn9w!4g~jdtV?Jh|`kTx<Z&5AafcU^EW}{nSlF;{Y2L5uSNYz
z0qSBnV2)ue5Tw$<)p%c(aGt28ZP3HEYbZvxg1W#}+$OQUJ|rY_gISO}lFq2KIUOCE
zn-q2d5J|*!?Ik*Pi+0I<a|f6cWwX}G?DL=y*O*I=(Xn@DNklHoHtjBn<Qw8z9NhzE
zrpQ1?7h+R-%U9h4ljCHOx$c8<Sh7W50i^cKBh&(1XLd$dXQiHXV*}0hx!S7ywnp4i
z#Bs5aXhvRJSF|YTjC7+_i<bq!<Q8cwUOfPj`xKI(3fXX&7zA``O)?|vpInmQ`EDW{
zC4eMU&^V{@FGr@4!{TlMFqu_FoqG5bM;<YOGuT)W&W{^@2|~E%`BtFw>!>k$MoxwH
z+=MW78&UqGySyN^`4C|4yliuM20(zPgmT?U557aNKTc1_(@9$5t;WZlI5kHlZW^(u
zPn#~c7oHB`(%@FU8e5BYR!kzy-_sir<f<TMh<1<Zt5pYE*jMadkZKDrhOna%;tr8i
zb(oAM336Y>rHS7Ind^Xp>AaoxO{0Z5`F=fnjJe8zAn>P4hnVOR<?^9G&yjWIgGwUU
zrH}|^Tt^t6NS&6sNEAb>1rg?zGYIEpvr^;wCf%zH9^x3fFF!(I7l3(P?vZ9X6vQk-
zMWi#95ba^bJyY!8Amvl$<wk6praN92?eq}yNLg>)!$2sHydD3faOM>97MTmJdw78m
zJ*Hu>$`L@hSJ!F`uj2*yGaewW=uzC$j|9qnjgH3U4in*+qsxb_Rcxu_@IL|MW<Y0x
z*_-?6V@hIqKN4FAax|$GATY<8>sT<e0Wm7e%W=@|gV{VO?I-6w9@yOJ;~widkzziZ
zhQm)FoZD6wP8gI6am82@$YlEy9i0_5jy{=iy+*j-N5a7M%qjnmtg{Z2w7A|j?gTwR
z&_#mn0z2!A1$Wor^i222j_&l#&d%T#G`K@>celj}k^mtDcMHC_`|p10seYU5`~K;x
z&eQdlobprWoC2h25%ts2z80mS*~XfA$f*!6G#8ldNpbl5(~2~j*<F?qqPE8FX8`6-
zAobW1x8c<oIJ5X%*Clf49fQKm=K*=e;57W<Sp_5qCfY)eJR8WIU_SwpPG7N!U{|-N
zf-_CDrG>zZQhRGu&mrvEA@%aRz_IgrfVEM=HcYRCd{@Uxjk2`ju)6X2#G~HnntY}U
zAk0J#xY)SdRtH$3SrpzBK)yV$7^_`K*zMDfLh+>{&5ObQmTt_D<VSGi#h@-ad1{TH
z`4SMfzGlT}5^c_~4)9)ZXWV}&P;F?AXKH+|4#?B+K=fQe4CmcgYK)bxgrpX%mgD0N
z5zc_^cqx7no%?*c5v^PWV20rY7PD7fvmGyxB$AwRjjJKdJ^6`4yx1F82&*xtv2idC
zy%xX~rkUyRxP8E#*>#|~b8A}MLbR*bjCPvF($Rv<6Zs!yuAkw05a);G4&HrOn>G|i
zf%t^C9Yi=44xQ|bs&VWMKsh6hm$AW(^)%RUBtLeS3zJB4DDF})Hw1SGIM#SXh$|$&
zvFKmaQ=sp4B~67zZz7U2mU>KudQZ3+h=S20%5ks=Hw4~t;+#6!+}IXR-dcRE-mK<-
zQIM-DjWAAS<!HGbj9#T}!8&&k*0AhXKkpLd<U8pYCy8{+u^Ud0T=ClCoq#T4iA!Zo
zM8^p7XXtZA<t|9ql(&Z2;GQMGpCFfmp*Y1C<Z|Kt*01?JI}OakQJ0OZgc63tLU#i@
zb)F%JS?+;!q0sDQE39qn3Ig3aY<_qNUbNY-sh`m}C?vNzQv9?Nf98HL7*S=u+x-E;
zE;P##Rx)=9fbZz<@#0P<P-O3l#UJ!%-K-i1D-8ceC9}@a4;6`&8p~YWe_RJJ!bf-T
zn1_jx`vqH7TwovYNb${N25498^(X-K#qGA*WRgf1j7zP_Z81fF*(7iCdS^zB@VQ{~
z1#vVi)ta>2V}RzoG#Iq4*x{c*7C8M*D8P`M?_d1kqDhWL)Fp+$0_1G%@mCRMU>CB$
zQWfQ_3x(NOQmG>=(ivTXx{LFtvapP`pCFoB2X}tem{O<HRr%?nYrLo7F-HZUVFp=;
zSNu0&H^012F7;HA7S&|mEZU`JTF^%L`g#nOg1%VwKTeF@OY}cYnC2RvnrpphpnMpm
zbF)ljDPu)A*_RN_zpJp>N^7K}H*cLh>tr~VV$!&%C^MNeVA`E+T8A3ZB2a%?<2hnc
zkL=~@X_5XY&w0`CJW%f0df|sV5GFW*<{d97x5tH|&8s1afn(_x04S`47O%cYnCi)C
zb4#r85>)0~Z+~ngvev%I!0M>qAr*Y3Rbtmyia1GSj+^q=KwK!yDG^6WOKzmVTw^3P
zIt;)0I#6wG&R%IMvD+J<Do}Dqy!@L`6je$T|IOhtH(c|*1)e7<i6pdoPOW2HgXx%w
zDVH$5*IU_}zU`E00+`$ME|g0tw`hY&jCq7WSD3p;iS)<52V#y?abxs_NVk7dUan~%
zEdD;Y>w*DPk`hqE9-;oYi|deAKY+}gS|+Q@e+W&(V7_k&^0oq8CO_t<v;Lyf9gAXf
z;$_XhT7Z+mlDIJ*5KSGq7K<+x_E635dZ)pKR+9dxO#!kv)^4*J<391Xs47OQ!Y(bF
zq8R(B|HA29Ed3c_8ig`6<J;wP|I^4vD=O@E?O{KzTdf4SzwrnS^-KT-lp6Q0`CCcw
zZ&BD0v3}$6>wkO!;QFJP-Z}*>tV+E5B_LA@Qx>;GnP`;V`ix)k59#uPQ4IT<a9-;%
zmrQy%_=6Y9@>=eQAAT2sr9zVrN1hG8Ap&*QA(=VoFCxv_CY}I^PSHgtX=-QZ+8L7t
z=J~@OYLEQ3h=yQR215+Pvjv(nT3Xrj;<XmYZ0zidCq?CSl3iAOB*YAXm2A9!ssmF>
z^0nXiony$JJ=uXDEdU-2G0}bceet=}hIugH2LKlV4??Jx^~S|N0=Wni$3^@M<tE1z
zKh_fI>dSv&>?XR_Kodrfh*kkkP?9(tKN+HV(zU_7zZU6DX={t|zZKuhzf(sXBfz<j
zn;0jH%A*ahG`#0I&kW;h8Ft{DawrxM;AZ7W+L1M6FB0m}4{O;`bH4c7j9@SVFMjc{
z!u~4h$kDXS58uxM>e}$~DR(gU&I;hcqRo6KQ*rXK7q`yFA5I?y1g1>dZpAskTy?fY
z$TR!m9YM9-2ao!t%_APu=K^-M&@996ZX7f>h;v3{2Gh2m2Y`ObD+DJg2>*~wc0}{M
z5N;3Y%AEU$;yc;yj)*Dq0n~EjJ|gc#mz&>daJcD>;UbDS7_o4VLKz+xoVz1?T0M)h
z;sOOkLO>m{x1ikN)4NPET-|v=Kv*kT`Ed&u-{^fK*#_Pq$c;K-Z2UQ()F0TwM*kuZ
zx#9y-$eC#|04kUjo5qQ(F-CH%;{>>rHN(OM7Y8ys<>Z4JUa$m6(foQh6yuI1!OS`7
zGj%TwNhx@*u_Iy`h}`ff50vBHVE|M}FS7h?+2T)5hopW@9pG(Ww&9OwK&(u{%NLg8
zlRQh{buE4tZH`NHrS}sxdgfgoSS^e4d@Liv-{9V@678bR=+uRofU0p$9pvsRO9Y+}
z=+aBWt~Z)jr~@UeO5;iaPK)l^$YWoePNuF5Xig%#lz~xvCMeI5%<vo^wC4XR)SPRC
zbL*}0|3%`E^t4bHUwfulZdGX2jj1$N6ydDcicxhYUO{kJC-?nw7?E3SJsjMbX$$=q
zkxmx(Ref=WXs6UDiyn$MazqgYWtWL@y6FGs4ns2XRaOH}CBkZM;ngABX9i@qhZ)Km
zAZ{azIlSpyU$9w+WCIHp2_9w|2}t3!haIV~RuHZ<8wlp9b3$At%3s0}dH7-dxLTmg
z&qgxc6W=Dt88>5rRogKBAQ(nuTf-$sLApz8x}gK1|311%jAC&uh0V{H0!l{Z6!wj=
z1(d9pf$kd*ln3O5Sbahfp;0$z6mt_n=3&yKta)%hwH}qXR+G)gOM;vhu1+qW2<7%d
z@o)lfDnhH7*(`p!mcw$Gd+OSRb9LxtY3n%CI$&;-^zKbHwh=;;v0iq^KI=l5bv*ru
z$!crd^5^1<X1%F^7sU07NRzm}RR5SJh$7&qAeLOe_&_T)&3@d5AntFxG!jQ`1WArb
zMANtlge!x$oKh^ZDKsfz?H)@jtllK6>}^GO1WL@<71IRNX81N_81b?Yr_~|b`|0%<
z$Q{%H58R9xq^{%Os+$+FTL*i(qf4|iXvP<EN1QAGuCpQ`m3&=PUXJ8MO|QYdy`^LL
zntYb6pqxFQw8u`O$&5R-cwlS7R0Y%9O1vY&5t$mx@v-RIfb*lP|J=sk;#nhxZ%f#%
zUmb{hMUg5`ZNxVU=ho!jVyF^>+krSK9g3tXc$XmOhCVlL+qN%gG_uOEv1p1WClj&d
z4iK&-FZyup0n;|w3CwBswQ8S_ZQ#y8xiOe(c+p7!r9wS1?pD~R%Osvimex9E;-8g&
z$O^3k%n35V7QY%>?#e&zA}wrDJ`!15K4rp#y%-HT?k~VDiM8c;-$sxNjBW+e=$7s{
zZ8tEph3i7OTo}D4i027}!kTm8-g^Qa4by%(<r=*g2({2#sfQ_?TN6)gG27nt*ph`-
zv91JRR>Bj~=zR#f`06>|iuAW!`?SaG`}%Jg1~C_JC6a7-+ajha>{$RqIgXEM*UULe
zc;+xlepBOh0q_o4TsihCLF8J?VGRz$<9T6p6LgljC7YoUD)%XN2pXi93nJS_Ch7+j
zcJda1oYMqA@@dOiEjy;J_OH)jJZr}374W>q($YNTB^UCsOEa*mE6?3=Xq|4DFnVPC
zvxWb2<4U+9($NtGw8pBf#owrZ#Lgn!ESl5lTwe*H;L^-Eu#K>nPhB|6tb$ekb)jxw
zydmg*nGW`mK-Y0fTh|~@kxwdO;RQA`x}_b^*~v*V%VVdIJll?r;T=UPB+N!;O93u^
zr@W{AAlfC$RzzC)e-R2l<#8e=cS2Bs4z#L{Qkc{mwG71R`5(3`aZHy#;n3S17j`=u
zhdrK=Jfp{dA-NqBb5DkF)8c+xD{Jg3$Q<tBO|Noq5knK74zLFbq6pnwAbqE>86#!=
z5wUr-h~LESPEQrSEy%S*65kW+_d~nKktu^225%(H1?j*JNqy`9K~x(_I`eV;e@<SQ
zvUG9T$Nw%UmyR*}K#>BIyv&SZ_NhS5NZ!ln&E;6Pj>#sfQKyt;2c1*$lssHSZhs4E
zDBfoR-Li-xbh9|e5Qqzmv4_T6TqTGC_v`9wt^JB*=z2O!kj#q$-RVgqc_q;7@w#A_
zv#W%txB@n`KbX@;-5b}xeQ}qd+?mG2e?*a6i*C*4IRL^ep41Jyb`)Kkq_mfjc}k~1
zw-w7TbKuDWiv6P>G3QcMsOw6Lck;UW%m)^TgtAoRZXv~&-Yiw4=imZGeG<tm8c3aS
z>LCT14846XcB#<X&@6G#i7$kxn<qEyBIslIagJ9GCG5%!RLZE4X}$bbU{OkmJ|-Rp
zVjj?tOoyULqG7_ws2)z3v{6c7w{!%AkBKz}6c#v+R0|x*A8v>K9u0lYO2dS?m&rft
zHX_|Dh~CPw&1yTUfFv81l<IRq<}}@f)oRJ30i1GsWzf$AE^rJOWs=A@&JyA3RJw-n
z!W$<Z2jW_{qD_kPM-IiG9bZIi9K`=WJ}6HBp_YuycuHX>qJ1gI+d`ZO8_r&Grn{eY
zViAp3`y^~V^<)sUj>Gq0+*YTX&~A?(L}zBBXHN%M_ml$R$pu~dd@96gB!5ujPla}0
zFFeA~O1}warQ<1sM@}Q?{#@;f6;6jTXAwiN+wP2ZK~A#7d(g4-83oOx&>S0{4NY1d
zQ=(l#GZ@t~pVwwQ7tE#BbJ{4$93{k!*@+yh<?lrV9{tpuv%&cwUP4(<bdYQ7>ZiI_
z2sa;luvm~eC+o)r#phUn$WC>!3qdH*6x4eabam1@aKwK~kOvLc0L+YO=KE174ag;W
zEPoMUmw3QF16IEngieA}`BH2ng5t9TtFXt5bQZj7gep^4oGl2JA|~gS=TcF%4M96f
z_MGlz>07x3JWm8{!rG%<Kpr}hcpp)ZfW(`f`(*-Lf2L=;%pmj0@|Oado9rZ_TcqP~
ztw=P9rydpvN1B@BQ3YMx{IW^>a#;b%2cNtOPP-h)Wn-U*)<#?b0DlHqy)RT4ZqUH3
zG44tTv$Kk8X$mE#v<h`ckku$6UsN)$@@K42V`I|Q5IF;;&QhE!z+ceocsrOq<UPXN
z5Xe?_d35|W1yYrN!me2By5c*jhoA;A7`q6gu!zZHw(E;8y#I|kg_r}ZMQJ0oVLcM6
z%RE~d``kbzm#-0F2XCO?2tb=6^kFmbt|)Sl&xygC2)hAVIIzXdknnoy)Hqf_mtRif
zu$*bT6~uX?=!#A>Hcqz#IVWUutZ5B*0OSRxA&$PY_(Eq>=}5mO$Y1Dcsv6%-1IS%~
z#v64vB!!#Y-yH)A*X9&<a%<dE&}E5JyYDUNlAdTeMYM~IUli`8qV0YVH!!jTM1EDV
z{p|-p$%D(ZIO;*d6qs!eDem|WD3=cXo`w-|g#ahX@lnQ}^E?dV@~*n-s<Em_&r!XV
z)c=ys#&E&r6~`Qmue#!hM}U+o2b+%+cDdMxq}TP}=FE?Ry8ELa(S{c!z^MXV0^XFC
z!tZf)q_Mk-5<rw5E0SZv(Tf$@6TB&u4B3yzTK^)fX{9OFR?ro|4Q*T}(&;l$d9f<~
zD}*6}T&ypeABTWH&1?{xVn0!Qi$~(5yZ9ueYuR10%Eje^%t0znub(c!=~uaCWz@~^
z?;?th^@H)mQ$;fPh+-{kiBAQQ3>zPGM@VM;(*RVammBJP{#zt~FAW-gx&W6$#%EeT
ze-q>(>4)z~V~b~sXj*wCs60^+d?`1^rS*T=4fF3{yi$i)KGDv_)=-eELqj4RjwhZ4
zagB9hRF2i2E3%OimEHh2UyxhYUT{^%yX)~7-Moj+%UXi+ge0k%Z)D<mP#T2UsT6+}
z;Sn~}9XDr+=i=y_FNtzC^n+&BD_U@c7Xhgv3Y7nQiJ*&G8qmgLua^OG<6GV793hT^
zh?^UkM+LY@eaX#Cnsm;371Zq29+;OK`r=w){<NwoFXA--bwF&ymwuhFi_J0aus5K*
ze_~v6L(NaROn>i9{%{SrB1{u<C8i5>W@Z#jUi&RD&kcHOD4r4Rn$Z5z=w9aS;;UAs
zaXMo)7Jdhe65$su{-toP0*BP3Z8PqFV88R*Tla*!+PnNi8?;Ft{MYxK49dT;)BA*-
zG4{T)V3_>_5Z9p(V~Q4OdB=hR-9?$Clj8ePf+&pk5hp8b-Z6>O^;Z25fc)hBIL3*{
zm8D0;ZlZJ1*&*W%xi_Z%E{?S3M}0&b4|8tbV;_<5pQa+gdqt5tzMVUxcRGa2>nTi~
z^OjElDO?F@_spOA+ky1{<SL?aQBi#Fj>83b3X~JV+Cw$VeOAQdVhR!0pa3$ZA*$F7
zLbwV#SRrfR+0@U$Tx8AzG5m`nO0|+6qOQhYg;A__YWxp{UGAa_MZ=fHN0h{i_nyLB
z;>P|m+t>XCxchNvFQbCN%T1CPKcYWiG`6_OEanN(6ghd~8unF@3^vbfCeIMSznI1L
z#@ixX8XkaQQkUiM{NMPKNhMs_>lws3z6EeMun8_cTm_jE1Kr)_q`0=tcVJYr6+Njg
zh2aV;Qo~%G|HAAvu2I;Z_V?uf&-)&%HVWmK`U?Rbz_?q{iAwq~OZ-rerhBz032^!K
zQY8xw^VWHFu+-6l30YrjXKejb@o77E<dqotGk}Kxf=|42MWYbP&-*sfqOhw>*G@aE
zNxu{jbBgYn;y6L>;@ly#EQxm2usGq6-Q6Bjeg$*YIxv{yrLNxqoN*Zq9uByx2w}40
z9qub<m{99YI=^+#2+2Q9opH5-PCL1t6<ar&35>SH2P1DUieRvCfJ0ughbVH<L}zoW
z_i|(%v}4AZ`P(Ju+s=4s7N>+;t4e$z!X?x~kH4Bpvw>(}v5>u`Fl8!DjvZ$&BI;#w
zR0Hmu1B7;G>m{dI&x_2%ZcyTZHwB2tl_|0GoP;So+tX3eBf<=)Jjv7MG(ov>xkQ%#
zyh@bk0we>BHKf|uEYcZBznm9x1h`@_aAaIukHFO<UYg#Rn+WEkv_u&~IFy(N%$c(R
zgMHOFOb`Wa!&dUpKM;07S{Pw`f>Pprb+AVYj!31;@qxhHA^L}gqG!J1%Vbl|hdk^j
z(Cn2_D(n7qAuf|vXZAsOuAU!^=1iv7an1roj0&=&)cL;^<}t#A+h-3;vG9VRp4ez<
zj(3RnNN(<hfXyg+F7mD?1h}Z!_IF43!qBw_K#s(zYE%?a8Yb}OX}f6ifd_25@SKBS
z_CA5MB1&>yF?kUPR}Iy3y#}<$Vg(`z5|y3nk05zyS5V;gzHgLJmrovqc@(1#B)^Zb
z#p1*uqei{lu&W3<AxdF4E<w-*N1Vy|0;9yGfLyn3-3A{gI@gV-nVCG}--1YuZ2)t@
z(nWkKIzCPpox3e3mY#T2fNRdJtbFJ&@6Ni6lR$T+E!GrK8=ie^yBMAuE(>OEu$)Sx
zZ6hIGEZBC(XChrr1{&`6en*S9yD!K86jJ^UW6>2LoC6XxWPPl_e-h?$X-|m|gKV1x
z(slb$()&SSXV}RzJ!`KBnG&Qh4~xjfYimRkB^Fx=#N}y`oObq=ix@q0XU5w!A*79O
zzgT=#!t_w~3W#BM5ara^W)H&Et%f^pBNwEvE9`$chUqeXjS(Q`F{g8sJPr`xy7UdO
zvBH5+Ydj&)ou#E4Z^c#&TxGQ)s?H5|>wc=hq6Z<8U3PT<*QST1EdC<e46Hpd8utlv
zrf8jEStW-pv#tTCvCj=ftHO@JWjGrPiR_-q5hS6#Xe1F}X)<rQPK2}MEjAvcYk)TU
zHREj}(tl%8jAK~Q?w3`);mnqE@limoFw35%0xfB02rcS{XY&gMkSmKge$PcX7rC0-
zZ!{$BB1Nm#7%>*gwdNKHRTq|uU&n#D98L7hm~(tVOQI78i1r+!o2N9rF=ZSl*riCK
z=IEUO;PP|k;eCg`=o94Zbl7H3-4%}r6F-~UV!eq)8d!iciiQMG6WOA~q_rX3O_+$=
z=w!zVF+VsR=#eXptOwQs&5YBVd8G4}pggT0CPCX*d&U{oEl`9U>=Dvk$7(`dzkJt`
z{1_c9n4(B0H;(=@gp0z~uQbT%ObFG$%X4(CN7&WuY~qZrz57uc_;WpX5jTl2i~CzB
z-X<FsA6TJ{o5r;_0;{tb;~zQ^TAYHNb|2ygZtbeE=f+^}hPFG=RqUGrUBt59CGCxE
zHUTjc?5#FA?^<S4P}(iI&_Yi&Mhl=OXfeln3g<36fV|?f{GT3q8@5^TpY|>>T(nCj
z7nn(Syqhp**;4TuL=H!74qEFO7$l>j_N1Wvao42o<n-lvVXi?{&x7E|PzXiq#;g1E
z`oByuT9t3uqKGdeV^*A>gg7H~Mz~9FjXAdj@^Gl49W<B7Jc1F2>iyJeY$ecRq^~z7
zi*m=tXoLH-7q$X$xsV~SPl(oS`~lKY9)3Jnlyl`mBiZ>~ylwFbZ;|NUVUg`YoU!hE
zq@7ufLv{dk;W(VfIU=1i{hzTi-;Mz066X)JU`S}n&R~{j>K;N~b<!5$Y~jv;t5-cD
z%NFEvfi7Aya^=i%wIDZ9x=rSs0y=#c&|2@|LEmcpao6J0?42ui7sNW-Q;yaDQpDHn
z%!SycLey4tqY1kac3)PfmAQd-R(A+?b$Fw&AD2O0aoFx)3^N=nl;Z*sE(MI0r0;bh
z?vOO_U~IKVL2K6${}$~e>2N3=ljIA+T<LZOOe-UF+dY9aLo#^dT@lU&m)NqL{91^=
z;3@zKY&m`ul>1GkDJJa&soJ&o$4v@nURBc5sLSpRLh|V4^~ISYh>F{nxK&}N(u?P@
z&vHCAxuavTefZB+L*b}BR^Asn_mJe_S99BQjeu@#y^Yt=He?l_9h$)1Rm!{!Fc^mj
za34m>b(tvV%MLc46X~RpMq!HDj~3!_6~N4FwBo4qLd_tw@8~t+0tH>Tc3o_o*aq!h
zlT09x(y|7eE;M&oxW+zJ0E}So(G$ZuAk5WCT<q%TFi}vRU%1VQ!$iCJO0;K>POvj|
zISs`7JaZz#g_Q3A3VBd>f#esrKNsT4q8Ak>_CUH&_^#xORVM>D)20&g+HFPCKs+24
zKPc=nCV^#hHOBT9(b0+IrcAVx$0R526$z6@tr5>F=)(2cx?+44#EfpP^rGse&Cr^C
zfO!-Tq6)#h9_vm4%Ok`-hha}gR0YF7ROJU^g#i;BpE_fd2p1H6y>zpCsUT<1>o}7*
zs7@2&>^b<c)zZK`X=(v-&p{7TzxqL-lS7+6nZx{J(8;AkqK;Ux&X5uI4m=Ac2VjQ@
zaRzAQvF<n1V3+L&Xoj;}N1=9_09RNVUdhYkPjyVY2)2nGvB>^KJdXZ7+#Z!<m@qe1
znQMsnUUY8F4jrq172xJz^BAuk@cT4SR%HU%<G=#LLe_3$juDpIN=i3)Z4d&llSKM{
z5pEAGkhI39_Yr117?@FIALBQRa_!k4%5^EV-{%lOmq)J!T_@6v<qU+)VmaOsL`E1#
zHN~$YTp!$3!sPo81;{g48<&Q-D2urc1M_H0#uj#I(S10mt00%+$e{;>)P@h!cq6&p
zEG%E_QVj-bc2)mX{c~ghEv?n49#JHyS96%V&JmJ(Sz5QsG53)`6qqYN&X4S(MrA-d
z%5*pLg8+ZT_~I&{BUU)7NI9v`U>j9O%Cd_^A@S(qv+5ugpu8dU%rQWDmXW(3y*Q8Z
z(y{f3Y1!aK-Q$7GMfRGgcl`YX0Jk4I!<rL_c;!T3H;ztCQ%{D>=_i%eRH2EdfYLtN
zPsXhx%u+_M#&35la%zDgsjW!b7fT3p36PMo#H=iUSp^NW*z7dIq==xo6l<MPeBC62
z3|+xoE6h#F*{a0$=0+iAVsZvcq(^@jDaoy4hO>y|@*_rOht$OwiIs$EK4V9HjKWTa
zCygjqN8BpNy>_6j53z{MX}&xg(8cGHgK@OzIRGAabh);e@!TRIrjkCr7B)f<`7lw(
z2KB%DFg}$s!a0BEZ`X-yTU|bUc0LH)&szO6UH}PK*n5{^B@rZsGECflp~LI};=zk*
zk+`Zp=wgUm%GM5ilU*!;$rq{qB?^*SW2v<(=D!5O?c6YKWE>*WeB$Aj^aL3l?oOdD
zpezZHNImyEq=|QPO7WTy^AGd-F5NJEAt<k`oiX*&A`yf?jeK&!WgyO*^*{b6GEZRA
z7XPkJBZ5H`eR+{UnQ4d_WCbDAB-u5M6H%LEl1FrU!UTa%lNqiruC7O-cJxtA>2qH}
zBsi*fJ@-`DWy0({-p_w^AiuitO8z1@HhH~q)zuL0?Z_OYhhB}{t|`9dMTa<Cq@yI2
z4P~|OwP1N1;jxsFIs|{qLd{=U%7Oe>h`%lOXQQ$et^;!g>3Lcrk<pwi(4Vq}mWP0W
z>p`3a#(?Q5P&OQcLMeQ@ocSyhT<*5Tfj1N>Nb!Y@^3aWrt4qANScDmZ=i#_tWG+}+
zRqiROvHVS7W-@Mjx?+rI$H6LDGTw6Z3v=Olp^lNPW92|SA{Uh~k@5ASF!v*g`6PsW
zQ;_SZ#e-UpycvWR>+PNzyD043Boj$3dW^8lP<bXS$0KzF!&Hvv@vD&B6qrOL^*uC%
zZvl06`>N<++**)wInXWAJ-Rd$AJ?gg<739#{Ga69P4T!Wmq1rT49SMuL0nEYd=ehw
zTJMfJhH`0|RJX5Vy!(`<w5SN9zGzOwe0LV#@Dw4Neb((&?*hwn5XXl2>$D;c$4lIp
z)Ajx>%q8M(Q7_+}csB@rf(KdKVvBnr$U?pvZ@kZcFtD5AIT4;p%H7f-d|rT?lEVmd
z9P*xcU!YU3GLh4etK458jW`xUPIIylH*Gfyv$kz-34)<=4c8SHJOJga*yxXEKlC7g
zv$C$AmeFqn=EkMN(;cpPsEAhLWQPESa^E1({HABggIG(<`42Ev6943@C`{I@NnFm1
z7L_N=ejTq8P+J$dB+(#b96k4N5f7mNqP5QjxWjOL9V<Oj{LhA!J@Y04T-wI|bXtE`
zNM32CbaVVJ$6Aj9xi7+rQdyTJytgUTwdYjJ#f;{>V;=*|=Nh(tC|<bRd??(ha<;9M
zxKUp7pFplc5B43LHj}96=YQ2>@=TND|KlGAav@vV8Z|UO5~3c*;cK2J2(xyyHpS)&
zx}!-D6ubP6mOA!{qWMgUo1f<}j*#N8e}kL7+-phEK!EFJ=IT^`+EYd95`@LCqUoTD
z5$pU1A~(CN$2yE;M#>Z#=YkWSCX)ISPdjqs@LwqB%DWcG3I6m9fH^`hVMpR#xVm5`
z)ZfDfo6)+HAom_tye2fzdET^+NhOjFg&Trri%&HcWdgVCb44cXsPyCreWw!V33f-(
za~cOfUyu!GCZG2LBt0Oz*5PohdG0x(xfB@N<65vBUkS@I6td{`UW9}#^)5On#I<35
z#=xvU?tcl0W|3#}bzUV5Q`y|MMxO{bgRDT?xVXd+PN3N~AcgnX`85Ezs%x8a{8N<o
z$=qZ1mYCxyEh2}RU*`|E5Mz4^+s31Xs24HcF7XCocb#;9v&owU%@AyiLq+>*T2Df$
zrQZT^MX(i3r;VKiDM8*2?o&7~J5sE9Tr{O*d)?9(U)Mj@%CFJ?ogf!n%DkD?u=KFE
z`NI`1m6@7i@Errq*r}(>;I9R^a9T9kobLM`h?|ButT8m#Jnw_Kz`W?51SBH_d1NDI
zLGGQVy%|0Lr495FZ0u7IV<*IwAM%eYg^QUM)6SSzYju16m}0;J?9XD%M}%GFacjor
zqFfDBN$H6@3Gl~KuAyna7vd5kC+a{l5NCc2<Q%J7ZO;{5`zUEftT7!V&qGwS6Jc68
zHWf$}l5N{wBAhdVzJ7EB1vnoMw9a^1kkiHbRFa|(a}v@qi}gP#l4wCu9RDN&%wQBa
zcnX#;=Ki!u7O&;yMz+1j3d;BSwFi<DkEiJmcw=>?**}AF9ax?QO8wk{#Lu4>U=wGN
zM(%?U$9)0le9`-q#Z5UL7UnUBBj)(#OGuhvhzsy7zak7D^h#5_B7#5I631p=6LyZp
zdxEjQK=UQ39<WZ$^o<i}!Yf;h6p?#SHbOo|ka??HO)QD)ajJMDQDe;fEpg0SgjBjR
zURa0|!n`1L{}w^67;ntd`b_-2!YHuzI<tM}FijjC3VWht4#S4a*Ne9cc3I@Pqa06v
z4?ykp!eWg10m6Br<wAEyTJs~2^VG8p+V@4jFf&v8u$6y;blFja;+?yAUx?Ge{TmPB
z(%XIpawpYO!m-dVMH&cqV-b<624iCX`d_%?iN{4|*7WyEAhge~j*K>U{8wRDh#dzu
z=Wo{mB*d)W9|~}3yLc1qfZrf<9rK$ehYE5-ryA<kEnb;nVq%PL;bX=bAt)Z+lVT=?
zYuy4JVQoL+8G*2|QTC~?{Vs;xbI`(D&g2++Rxesan5)U~L`og62+K@ka3S1|muL1j
zC?R7IJqv_eH0|2+Zu4`&<jm`m$fjpCfeSNyt7Rhj_4?;*{Nrg%(nU?+tR5TA4r<<_
zv#A04{2U<8#m~BWb8^Vsz`4K-?hb9^66TNly4f-I#o_Z1%dD=l3%y7r+YWp+JgA^6
zXLX-y>@qJ91()OcIOq=$POFDUhe`b;A^xIa<jCZ#;~YV*1G@{}z~xz!-Z*yw&^+Pd
zW;tFEO<UoUKlWd!NJO`@QV+RX5ZR`C<F<uO$S^p1MYs%DrgtKYx<QC*-7jYxT7zz>
zW2hA(vH}|u#7o_y9LFe3`ZU4BI6*{hF<?)6o#@=gDAY=|{2@VFnv<s4W{VUbq!;m?
z5lw1{yzm|%!aa}ul#~wHRn4*}AbF#y5TzVOks%gatVoHw{IRi%XlI>0ImbbN1oGlD
zaa63ZI25hb(-MseI&KxSLEVyEBFL<$aHdICAg?X~M&Y24YAk7@UwwJRQV>?flOBC<
zGHg$Wqa&Agw2JQiZxLaR%hN90m?nhURB^yQ+c3fu&jus&Ws0i+=J*Xn<tYd&rQypu
zs<uElfe@7&QmTQyv563p9pv(PFo)^IF=aX9tz7ucx<c_4rwb{5{`F5Fa0DNZ_bKT7
zkR-)Nb+W146Pv8azuwYQT~r+KHKD<Dpl~y|y*GXnWfqouOX*fPURbF}S4a8#RxYAw
z=f}A3j008ygUc<rW{z(}<jF@WWh|7ObXEoQxWM><C(0A2uNm&|pqa);KzN$uHiqr-
zfB+|?Wy+6H9U@SD3ER1&>R-^<Eax`~fCbrE?$)cBl!P2k-d!g#Ws5~u=imAvCks$H
zoeOt4-~p-5u|@%4p}^ueLV$-1{dP>u-vDjSmubC<$echrLo7O~0MNYRfOlmb;AWkf
z<WT|CO7@(eD(uvCUmhbyL%XX?>gM6+QZ$SOq0L&c@`bgcoO=hgd~SRZUFPiU#}=Ic
zLgGAn7<(w}#3lQWPejvlTv@cnyb~cvms0@kahWJr1I+`3K2HlE9jV!GvL<0?T4gJ?
zxkxuJ+H|_v&}n2ys2h+T*{UAJHUC<GX1n}nH4Y-y(zU=M!X3HFAFTDvT8jy!L`e9t
zJ=|E7Il+yDRLM(1C<Z5rD(BYkf0qUi3X%fgfom6EX<H4qZx-SrWZ(5%-ILoi;d%a<
z%9emSud+_@F<0npY2qj$;${o)|E^28*3vw@7$XHZg><6}_iq%G3tj4xSH?Gl<bI8Y
zyyuQP{tW1mhhURY1OM(9M(vq*=`^kDL6Na8brxN}NTRcX1_gtx8XE|75}oN5it@MC
zF^RrXHRs=;h}SG8F*nA)g?UY7deBym$1Pf;e?#D0L2jowv<Pq>*bqw(0DqhtfoTDd
zx0u&6Aq||^Y-9d$Z<BMKm~j(mD#U=CI64|c(Rk7cqAQNs6w<uo<fMac>>!AgS*XUv
zc_N%Orcj<h{<ImGnQJ*!?6f(Evuvxf`P^SLNuUqZ9<yv&{E5tgKSis1p+Sz!S`-W}
zaTU7OR)k$~e1bDJ{<JlKbHcrCJ1OraB$o%Zx3tPm6-I9C^gCjn?IFxlc{_m9Gy(6$
z2zK+gr)sL7ZM6fSi%q<~7!plc5VOR`3VW<gVLG~UM@Z+3dUYEMe0SU-%*Dm!1?$Vz
zI{~=3?B;Zf{;?2J#5Y#FzjG0NkST@dmIP$p%Pf-kkL?2FL85sCT}$aGR%6axi*M62
zCcJqJK*ALk)uz7`f1*Ol?(Iqet{2^>J02AcgE}g_<TuN15XJnE?mqBMEs*47$?=H7
zuBsk2O09Iw?qDt#Cap-e@X;jYq<e((9ljCb8-;W25jOMha(jT}o~V=cL!#Xin|k8e
zOmR44LS1D~{-ccA7Q`hYNS5UxF5AmtG(qF8TA0)FX9}|jms?}}-UQtcoV(GDJZT>Q
z&y2|Q`=o5a(dWc{L0v%P5K?5jRtT9U^T#_xIF;%EdUP?aRHQ;Lk`27(vA2X$wpO%)
zk7_JFkwVpA{8xZ`4W@i?RuiNvJYiJ4A<CTR>OVb;gJZ5TAVo-5(6@`o{av@`7=H+H
zAL*05FGJvoN)d_GL}&lonn9dvJKAb@iF8He|Bw3@0Zs%RWM=V7R9lKfI5mqGrwM_d
z%&@=I|3ti~;WMGNND>a?2XuTJ09@dzDMqvtrk2U&V_YT5jgG{#D?S%Z%t?eF>>%v=
zC8L6Bd@979P2SP8fsBPa!Calr?$+2zq!a8xe<^P7g3e5x7|V7SksDcv$B4|GP%npe
z#55r=m}}okY}5l$>nY>MkBD<81DG{>bz2M4AA9}5U_VRuY9jo9b+8kQe(njYKvYQ2
z8g9}@*x4cZm{O`lmyle2mA0ebu}9%M`dWqEEjfE*E}4AE%r)@4G_WDTkVA;m;NGSy
zJ`qh7QuFUMwTPbf#??4d2n><tRd0MRisCVS#2JHxoe-AYeKF^d|HCRYHr<b~3&YBv
zZcdT*Z(B!M4{$z~u8Q^)>U3}iGcgw1AKH_w-E~ybM7)Mz8ZO-ea<6{?Bqiym(>{8T
z!)+|Zk13p&Vh)><o#EE!5J0-$_%&iR1#=sgcok|e=06mKgq!;l$2Jowez1te=OR2c
zb>r7IKSPNA>|x**AIXdWBe;C&f3g0^O^Rz=Juv+bLGBjJ47jJ7=Lmq@ExA<UTDcsz
z{uPW|lW(~Bk1isl=i}u^d?R7*w^%mBcl9WogL;w1bHRU15v9!QRD&X2Qw*6A;>Q<4
zY8`YO>iCNv3y}8+6(%7`oZ-lJg54tWNg96_O&^d#P;WFH2L<Ob&y1D+R>YK7-`HKW
z^OW^79sVF8xhHUi$MhiQ({~6ZjRrdIJN0j*(@sTx6yib1v^UT3P%cd?9*1M@6QJ|3
zz$k(84hbO!k837o&UO;C8$nYPPcFs6g7UZ?6YZkR%qn+*w~BO%+&W0?apB1zxy?FC
zXp4R%z)`q!)Z04?ol+n;@zi$bOd-ye!ILg+_B##8wNF=aez0cI(~F4gmy-jG(wRW+
z6Ygj*dVzDmT=BH|zFM^7AQzx}vMV_LTre2fgsoon?}VK{0=}erceo&r21z{nc;%Vh
z-FblU!s;bWqH4L|TlLlRq3MYn1$j>R0*K6O)ZQr6dl!P3Gsp~C@;AN+z`3G+lk6Km
z5M=h!tKyc6i%(E$;k32KCH`MZnW!y1Dv}&g1Zs~@MC68Qi0?&FIF#LzmD4Yxof%Gw
zD^2X{+Aalhfp9~`6gqgBzhO&Skt2y)M0>0jX9GMulw+35`6GADnq-4DQ>e4(sdOh>
zn6v5-9&k7~m1D6hidfPKZD*mpMi7%s6Wv?E+`llCdQ_x~NN+^0G3iQxJV`Y(rjkC`
z@mB#lWBE|h;JHnZTb$|FZ&l27HJH<GW_DRuq-)1)E{~M?-hXrbk-x>3P8#m<t}thT
z@qW5#nCluKmrW{3z45Jp+{>72*)pMWe_erbj;wIt?Tt@`(et{v44C73!tN&Y30($V
zUq`s>kudxoA;hO&#m5S}&vgtk9UOE6G(DhU;^>KSjtKv$7rW!!{AUYq`@}OM+-4YJ
zVKSPOeV-Sqw2`d;^+v+3BeM-VaRl~)oMH=4cf|TP<>=f}E`GDaQ_(E>Rbg7NZF0<e
zYmP!|H7RV4aBkyGB3h;_%&j2TH;0RK4)|Q@ikC#Yn1vL-8s7-bU6oCHC4Ld$vclK&
zT;kHV6%h7xsTZ6h$ayu48XGT*bSkCRP891NydA)q(FNqlQm+_H7n*oFt|3O=0YS|M
z`s0B+2{OQ>KhFCsqMR5f0{YVjcNIXXT;ft|zi9w2qhA53#^r)spjL!7&xmyEC$CoA
z7uCN(KaH-7Pv^UfNV$M3-vdGc8+9YFwFr-htSHYn+X*z|5vok#l8pnTFwYO;Cq~1)
zP+q`_8*u7=w_q1gHkI#-%mw78AHB*|?*nmmop`0Ou%P??0+j7~J07F@qDNq!0sG4X
z`26V=;>7Xp6MYXsnm=sQ(drqDeg6UEa%V3j)i_m{8C@Znu2|qv0IJ`^SwB7#k*klG
z!8@Ms1Zr+aM$z;bVHcx{tp}a8SBMjwf}{*l{XYRnkcpD##6&njHd);FYvrE#Ux4l?
zL_%8thaKG9Jzk*9vNb_oLY+W`Ud2da^L+L1qm~eur_p!%6C~g%D7gT`zl+cE<DTRX
zvj(S3xZ^8Ff@nU!c(9`)m{Iu#otgFDV8kFlioB*M4=k>9dg8pNpy3Hd4e_?ZR9LP-
zV$uIVm>ozb*)}u?aMxhRfjJX(V>W3M>;#eYHN<lNg*H#Tg-b`zA)zEaAYRV@EJPjw
zN#6*s?t*GPDk&vx_*{X|2YL}jTqndFl)oEX+)$snUjTIWdSE_w743w$<`IYX5<(4I
z5mgT;OjnanNoJK7@?Ug-nBm174cqv+ULx#7o8_W&p*r1K!pqHNqH}4r#lvs?%LUSm
zYZ=ZqQ-zQ%+W4{5D@A(p2ou|g=07fA<2Z%g`y08~+WplcD*dSxd)8@`uu5@*XtP(3
zqDj2WeE;}spl)})za-<*cutsFQ-)X{i*T3X(j^~eq}njk>-=G+a$(HbUT@&uEztEx
z52+Gsy#Y;mkvYaT3K#7?A=*Tf4TdIdal)GrZdu9BxiN@O1esSd-9-qu>02OL6(+?c
z3cB(wsDD2o($z?xBh!vAg^@SLUa{uegq=5c<fYhMwBsj*0kPsrVU&jf*%vQ};A4!D
zqVFBT<W))vfAK^n+0``gSodopU2-~CvMF8ne?Vpp*K+Ol{=Tn3C9iACTk0{WY%e=V
z2F{-E6T^i>)QyF)Xs0}YL`_TmP9Ffd-0au7@M^-Fioz%=kHW`e3j32%8T%G`@ogUh
ziOo7CZ~PI2ha#H+q#ayH6K7Mw9<tgapbPvlfNQ{J72OEwYu_Od_E#pwlG6$1a!6TJ
zTblbn0dsp*^~MLjOoi0Oot9Ot*DrqxSX(2Lk9(nC2z9|xH13u^wb?%_(v$N(mdLq<
z)EFX#-oe;WP@Zghc^Nl8{~RE<wC3}*zbYc=N*hH+0YsouX-!}K>mmYfV@8gModm!z
zG+eu4wr?PErKBV>HTs08H`oK?mfu2>T=JYiMPB*NG`%Cu>DBLx1eht+wH_5hd(k!G
zONGs$o)T{`ql?!Ttv>*&0LU`p2NA^9ZheWL9Ok4N!+s&`dLpClkB>#WMR<J>+b9;M
zJ$@}P?EksX=R9@uZw1CptS)BIftq2>TCYGMP{+k|{yuglV6(fUdk`tXV7w@Zq9HUM
z74L}1MW8dqtTRJ<Okx)m^N6J6@?ddV4mTp?oM#qrmsA`{)6jClYCXkv-Mz729W3=v
zMI5VL(dn}ilM-Og!Rr$u+?RAG&efEjn)_6!GpKaY2ejv1dp0l%GYNNBCn)SbgLNOL
z)*l7<15{(Ud#4?*nH`MENNI<?*Zgxpx`6Uxr(x3@FAH=+%$4${JtqK#>qABpCy7v&
zwDV&*+{m0h^IYK0jfuD|8A{wMEVH^akc{E~HaC!SZ0_b}3>C;~#5{mx!2Sl2sR%M?
zrA~J#tZijS{HS2A9#_4+an!s3wGqdT6agm{{F_i`shhY)=K8k;`Ah8#tMOGm4(|ac
zo0`4nBTi-nvQn&N*zoKp&|5(~eZ4q8l=}rL{^+7C0PQh^`HuXZZn>c2r}-yZMAgd2
z2B{S-rmh&fFlc5;@@Y|x$$}hF+Z$e<QAMs0s(!}Wd9%Xm9onG2Aj0{wB4Vr_*r(Uh
z<k~GsIIb1sG?O+rt2Pm5T%?G|SkkbJT?9EbsU5Oi8^0)s>xsHUdps=KtZAbXja)OK
zY$uQs^|YdBz2ssLW-zBdT-Yu1M*zw$#p#a}CMGQ6YJ71Br%fNld7S7x522e!^!6n{
zJeAkZ8sf$=g4J)6wrD&rk_>PY5tEiI(qO{oqu6;VM`p6Z15}hmI#ZsL>W)(~ZF5%S
zb4%BwqdvX6NJr-_=!x;pGSIo5F(0FhXAdh7<R*C1iYJA*RdI{m$vbqh-m*Xxu?tIz
zsS4MUXln28iED*;v|zJsP3Vo519QeKS3C?Q+KF;G$DW+$$?6b-JE^P1_kzf$4~xSe
z6?UrdK@W+dZ?FQGr>};QV`F!bZYw=#jdiR5mn}Izz<yi^Ervh3_BiWL5Ef8!is2w2
zJ65(pE3F9btfbb<e}l%H)Kt`INPHE=auv|aMHqumR6(RZpbN@xMYwRSs8jW>1nI)z
zY6(^0j|4bH1`ls5#DOapU$kQgahFI^OSjhdW`b$FA^s=AS+f&M65ekFxhHdN(}L)M
zB6h4&B#4!MT(T-8V;(aW#`wI$p-7&gG8m7GaOoKEoB$3R0Z^-5^7$IC2yz@0w@}Uh
zMu7P=AR&{E2sf_==32L)3i0mh#W&=`8RU>Pif`<i3%1=f63lr`LbR<l?<+x0TKXAD
z{qg(;FgKSxMk4#Q#PxpR#T>)bpBi!0C{U`0sGuu06+uT*3653RU6I3LbNpR&t~z%5
z9KZ#@rYbq`yvk@OmjsUux``f)_3Bs~Z%C)-(YwYF&kc~I|LLv6{|O{tj3i>%Si;Qr
zJyYV3;|NkVx?3q$6XD{shfEGq*f{Mo-e30Ne)r&sgv}pqFtqVJUJz`ylr7E^=|XZ@
zuGNr*XH77t%Qb@38?<HKSg6~l2d$4=)`G0HyPk@FaBTq3Q8l4GPxHj$>l83&3f`0R
z(vo2F64}mRtgdJ-2NrsLX}5N}Fr7)UgYSx8*M;&I%(iS~{tWF=P?bVlH4a`61fF#w
zE;wd=!Y*w)JH;h8@L!n9Mu);A+m7e|rZ`fRS;TRk^|Bf-YzX8AVK%8&c@(rWHroi$
z1*F|rCt%58!d#?*q=&vSw0Xk86A}KY0&;{(V`Ds0ryJpR?6V2~yAG1aFt!?nxH#QN
z4^^!pK~$@qdzK3oCI|eYQB>BwOX?|b<0G$TJ8Vi6=gCofT>K{5t*I?3+S{6yF5Jus
zrPV+xKgVni;Ebync*!gLc3S|sbZiK;!0jT0?#F#&W1LX`yNcgLW{qh=Tz{?oNCUP6
zfYp*}SE3@yCBt*^U^Hz7?O};UrF%vjE6FKagS+jzFn&*@ONP%B)?@j_FJfRk8Q*OK
zS<4qWK&-JXfIs2Q=Sna8z83|-#WwCT7Tk_-W;->Z<Ygf)8rPo~f{S(nZC&yfiD!m-
z6oeX`vBUOGkgN0fU<bl3zyJ-?9?R}nq=(rg$0^Y;j!AfI9KBN!oAWG<zU|HcZX&%7
z#tu!Z`y0CyX&^@Ki-ujH%@)KuahXVXOAAhHh&x45JtPrv!e9JP6UKx*yqqcGaj$-d
zD3`P|of1dw2H;fN8aty|r2j|ENs~{i|F3Rxl}Iy6kAuah_4r&&5dXD3h>-hHXLow%
zdCHzZxq8TGI&hcH(dY`H9%Z`6;l&g-UI}#es1lZX;#I|>UacH5cHfJzhai$2yLEY1
zn2X=bf$gk)AoC<79fYjj@`_*=3h`o_)`^ApEg&9jqny<^vs6H(uBj*)pdZu(l#@sF
zuxp)c$FJN0BApwD;w;0%+gTadJmPH79ZS?{M&zitU9?M+jQAMW4-3lEUu$-X{#gag
zY-1BRl>sI9N5h(n#I>u933Ymsq%a4F&V#CUGa=Xgw+PSEC3i~nx+PmcT(?PB{TwPX
zXD8iCJF{544U~?{Do}}&L^us;7SY-ZP6BaN*jqKX^>)Tz1UXrjlNfA=bj^CCRIZ+V
zmoOb*s_}t>Zg7mVIo$jvz}<Sn=-8&CPK}SUC>IgQ9tGgdcHWcg1a+y(-FjRgrU;=3
zW35Zv*HuKoFii?!@79q4d`)mWcXT&#$X51%?eU^0Cz*_UDzRn{2z+SK2ECLClo%^<
z<7ECZ3pz_&ozQFM?FGuMiCg@%2YF4HlSI>qYs#2b1<6(K=8`s+?gMbYVTmcl`l20y
z^@f@b>j!ZC@!KTDam^l$Lg8j_Yb-W}Afsg>Uf>SMVLcza^+1u*APR~MB>^s>+L`se
z8jDRWB4N#f9@hi`E;3s%$y3f0;u4~5S&2(TdknKvXu*ygr#E*9q!2R6y<cJHQ<C|d
zT7Q;7Ff9rw(ebdJC@0z6KdCLIi{@*bk;hU)gk4{Hmpr@5sZNtnVsOZaL4{oadBL!=
z>N5nA!RVw(e!VEyLJmi1>)UbylTc&7N8vo-aH~>^X9c)9)fMy%$m07KDKriCAd8Nr
zgt%I=yJqZ{V?|*qIH|6wFddL(Pq()>))SE5MXV+^Jb(!9<<-I1S(IymV>P5pI!jCw
z=#r&3k>i{L0mN5E$yWyvb}4%+T!k^~uY7O;=^|6jvyC7+tt6E(`Va_bxx%>E`%oxn
zsmu4aK3ToqBhXp0igMfjpa2r!NUaC9O|XWIiDgC93Q%p5cKZr~^4F|arMOXm^YDH}
ztv|!zfNt-`4y*x|5bfsa>rQv0Hwy6tQblF!rTPbDM1wKuh$4X&y<K#MXxPHYj44MJ
z|Dj3P6Bmfi?UWpqGdMesDlmD~Pgmy$3Ukq0n0>JhKe~v7U9XNxl{iya9u7#3D9TL7
z6rW+_jtu1>0p>Acx<!wLa{V<qa&jc);R2o2K&vDs8w#Mh%xLn=(3DA*>mjZ)4nK~6
z&1b!>t*hee{su&a@hKfE98Xx&k{%Y=Rs^}TePg4)y(lvpAwpZ+EZV$~+&8VvKMHdR
z#Rp_VlyHd?09`_lpw$F8mR?4;&GaOJ<_7038M{6sBxiuVCQ_M7ymum)=B6nz_elg@
z9tIwF3a1naAXLV{A!ZRm<d&w`AqUxKopmaxbJq*%gQ%pT33M8uIwr0S<6sf)FsSCn
zc_LkY6x(prvhwKwFidK-t19e5N`s(Du6TA3Mq*rbmEv#_E+*4icYGn*^-Q~1PMAWR
zekXVL)i~lz0Mc(5KOx4SMc4)7VQCRAIyR8pg>eFlway0gq^1P(4rQ`X*P)3c-+3b4
z?8t5P2tllM4v-7hg4I$CiuPp<mQ=0W4`h4APxJ>xXlsH|o##Th2#EAhdY7NK8wI)s
zllpPu8KcfC0G+i<(I>$9w<3CzK>jvC=A10uIaT2<`+P8J&gLzKUr?lB8O#&`PJ`!W
zl84rb7Xs13ZT&GtK`{+`+HVv#kGXx9j^q{>fp}+??s!;TwB6WSu#=RLjs(E>3CiDM
z%RqC?b1?`h5AibD#tNIQrKyO8L^`rMJ0o`YOF&#Ji7)7fWn5wSqcPq-{2~&5w&;ps
zhf5*cdS%u~)<yaEoar)9vTR|OI;gNY&ZsX}()HqH0&9J89DQ=?<^G}xvDc8o{sN&<
zQb|P@R-nI7#+>%AC}JTH<&dD=`PTv+i^D8e>{nk2;POpDlA3$hf>#&ONQ=bA3os*a
zd#1DNF+!*U4uRrXh0W%Yo|a~HS?C)7tsT#Z9G2I(uN8IyP>^HVneo~JkY^iiRs}dQ
zW-zWQcs@dq1{~J&agm7JuVvEuK>icmbmk8cZXT{dyO5%^MD@A?(`g9{mvX#R$8=?m
zeiwSE!pv6gn52!($BSGKSTqD)B_|5VbxpTuy|F<(4rV6Yc0Ey#!<g#R!z#<)P<+Nj
zhuJC)P6U~!vgBrX>~$lEMxw3=UJ>D(QCCN0X78H-TyY++ug15c-B&T2jL|nkdVEV}
zS$}N%ulmFBu;-3P>m(O(@!Bo?o4X&<JRO&L=t`h_Ma_VOX87Z+pt<eX!{9AcK<?ah
zohA-|?9iHo`a}W5YX{le{Sm^c9YV=coNyb2D<jL?I7PGvFlkhJWl~2S&lJoOWz~=K
zMX4W1%YUWYA@bPtOgTzO9v6B!yiYQh=kN44Rh+TJ@Vkl_>K&|1Fm6|Yt_7#Mw5rlW
z%0kVx9ttw+G)UJSAL4A;Ce{HgZ&;tU6TpDrX{*>@VTz5zUj)h*iE`bvuFFA@?!&Lo
zA7CaBH!JMQurKC7AD;>#LuL#gf?DndaP1I`(QDN`@3{xirNq5ujJ($$u^DJYP=Tmh
zfZ04K&ChM`D<X83(3*=r0nT_*e@7cD!2Uwq!ISDByomdYR60w8rFdR6?ZA6>JWBEa
zM6DArym+99PH>=5^Q(&PS$jMvfNyxM2D{x%Xwi@FGl5+-E%adV9dbT)z7+v{$IT17
z8xbxod1GW9FV+!eXnB_(dWbl*Ln{THrm#za4>s;^|C9gZedL(%52xHL=Mfu;@WjEc
zlB-Xcd#W(kOjk;HyU{-TJfZI1+?poSOlH!KcqAu>ZDwyQ_b8EQ7w$T>*MCT4(RNcY
zd1Pbxvp{Ob{&;lk@EC+k3fJl0o&RzQH09Wle~Q+}i~mv7B8PqfHMY}h?JTtI2~aZ#
zch_=TaEuV~U2F0qBIuX4vHCFo#ja;$9QP!Dx!JR7Ykd0@kUNiDG{nfKp>vz!&a4tY
z3dj*;;6Cerq0L0ipXs%(#f0Ua&88D(EF;8UA>J4r>poM&V1pxPQ5!r9ViQq&1!ZFL
znRBZCsSPDsL<-$>p>v)C_AqGDeYZ%uKxwEgR(#%JULlV03cHTIcw18&Z}9>U>}f_@
zIPQKC!e!+l6e-@_C&Xn%H>w;@h&Brlj<Vl=Q-FH|H&#i^oBIEc!krcOxH6WB1z#%C
zLW6!>oG;o@Bn#4I^QD5A!&#4HTiezs;kRTK7rx9Nt{@VNcv__U6I=Hr*}_2R6~H`&
z`nl*Nhu>MS<7%(K5Mi=<L!j$5Nxf~!SE1dT@b-(xz9jP+5bD};QO#OA{xtx&g+~x(
z$Tv(NXKsxy5oVp{d>ls3_GW>g0g7Q(>?*{`^>k}XzncKMBYnK3Cw>-1;+<`b+y&o)
zaDkZZcw&g5SB{m0(y(~ijopiIQ+%T^pYqgCU(EClgn6aMO8y|Sus6;IxT%dc(?9O6
zI2hF1oILa?%=KWeEd!rpg}9lLyBAvHEI}^!#4)k>yQcJnKzl4FiY@EdHDWIX{bf~4
zINfc3LAfz{xwfKTRO8s3KqCUvmKggUQF32q@7*RccO;Zju*He}1(}bX7{vF*bpmRm
z0=GfTI_kV@y<fybcMMs@h5}q+8r_BzogaX~(rRfS9{RBOABi$b(0KWq^CK`=i{eOI
z%rBzWWvfXEX0#v|S8qveDAHx)01L17sDD!GU~@>74*A$grg`pt5oUM?txPBTvW^U~
z?z0t|Z+ekvD>h)LX$tU1lv7GcPu8PIJ$XcYD}Zt%)$NVhK82u?4ZLeyQP@3efM;>>
zW~j|y^JkzmX|k-2k3>0rDI~H1?)@Bu0izoqNl(reP4YOk?v5`-xY9bxlwzqb9h;6l
zrLC<kb`hN?#<abmryVOSw_=GUA#SMy$rEKfA;6ubOJ^5`5l51*ilio?;f(|EO6>A=
z5mCo-UR>iSlfYakH01D<uUIfaPU-ekI2WjusZWd4*54Eu`e9n@s<H96U`_;`CRY0k
z1?16SU+=N|4?v{LWnFYD>~!UYhif_PHGc&2T!SdRFU}Y3DHBJW3=9@DG&p0zPvEdr
zlX`qD!Wk!P>X`Xw|I36}iN!>?6ll9}i0g}K`G0opJh&jjS@NO_F7oT~H5X~1KbPYt
zt*W;~WOiUIolbRE{aWO~H&mn=(*#k|QLLBO{YE&m7gd*z7(T;VwSm9`ezD|?kp6!X
zCjVUN2=M-#Lmsm``<2v{{x~y#xU==(_Mc^-NeG;vCy$H|M0m29FfOLd0_Bo2@=INM
zE#{P2!7_u{_eRrf&`waoC^oA%2%+GO+)J%KM-dyjel_+I?OulaJ$k3wZ1y<;&EP&x
z0NN(cDJZu{xx_Wah60>zwUtfX-lCn5{MMor#B6roTtz}HSm@|*al<@CJTBLhO&01B
z0zHlBG3r(3g(Ph@1j#~Y9no-vD)+_SBB)I(b+0JwQlVN-FF)W904}+l%<E0%;|1l`
zmFB%v#qJTNPRrW0@qEQ6Jg1q8J#u~^*R5Nww2lx>{%H)Xx_}8DZ|sZXL@?iXw8V1?
z=7UdmI+z6Ik1Yu9#F(3V<JN_sT||0T60ChH$dgj-<#_t&a#4KPSi+0eXQEXKhE^Q4
z2!w0Sc9KqhhX7THCE`B{o1JJr#wLqGQpm23*ljU_q>(!GaUyb0L1&7!Xv05(cqvJC
z8F}Bew_vA77l~s<I&a<K@KX2U#s9r%|NOg7;#MqXU4nmIVWdZ#ys(NE<j%s?L|%&)
zToTa5!8e(1h?fvTorc(8x92eaqT;Egz}=*^18OY4bn$g#cUnr%T?WMYqVvyOS&Bym
zxh=>7qn}~Wu2?(W=~t1pxv8(Ms-vBz`^}aGryP3r`0N~>gx%}O%YnPyu%U%DUHI3J
zS(gWO<q$1!pQDj5n?M%^!AqG(IipLEYthjyp)03?=Y+X{Qm=`3R)jXIIcs1RGxJJC
zIv6M<rR>$}5C@mkp)MAb>%~~TQY0OqwUq};2-ao?HtB;gaa9m!Iawt=R<yJ1X&V|U
z#oYp&m?r8N4(aM8byCf!v0fc)Bh?meC3@q0f&P$tkhoD~PB=XfxYP&$^CXQt*0Mtc
zIS#JZ5U5TQU?wReM0Vvkay2lj(>oM@S1^k`(6dMz=d|5^UVk`ke48e*+ZL+>Q5@Ng
zaFRPDlTrs^jQ=&0+KqA|yhx-+g3i)3S=uy?TLaj|kqQk`r@=^=v*K!6rySlyAsWN1
z)CK>35#c%|-&xT)vH+&E$?xw&0+{3x5z_155JfrlEENxmFVFz#PR5xkIen!|E-I8R
z*~xf2Phn?}+^@x)qYFS9fsL`KfLfa&9ZJ2rQ6Pmx9lte}9s}WuHOc{^&b0dpbIx3?
zu(L^Gsu&CE+;nst5sl-VPGgDAKTQPX;_1CuWP<<D<~ptx;m$xaMgK%d_qrkF-?%2U
zONf+2s}N_VBL%upXfop>k!|gJYk`pg9@(S3Heu4N4kF6wj82hGQx2r#cG0j^PH8zk
ztOKQ~Py2;+MYuGbO*l}^|52fd@sSA60u%+)&o&u-m1FjGom`U1#BHKTA!|hcvK}PG
z(lh2G)+g)^k+qDYOOTnSMFce!0p=~m)7u?V?0Z9~D=bwi+PfU*ZUE*H%i^VN+0Gk+
zP?K^u#C{^g4^FsmC=3IV0xZki!W%(56ZYQidP{&C(T#+<EO^#rExED|kOOTsY~y;2
zN)u{(69srQNcx2HSUu>1qY+?pk!#Jkq5c_%A*|0=*o6PZaxGAsig4>p()HnGqWy8h
z__6VZNSBJ^if#)wb$mULqYiY7Xex@k&scaf!g;A6G78E=bp*u*y@C7W=EU*NsdFHn
z5=BvvoVCUuwt&bpG&UA`kQl3{Egd7hCHt)i7mnBaVwtTVonLw{wkzHf;+8^!)YmE}
zqHAwc#F923b=X~qWAOqHcTr3<cM5YwrO(+FGi?Vz(R2?R{UTg6`Wf#o>MnJ^Iw&6n
z*uc>6*9wFmm~6!o+ZSJP_aP5)RA>`{W_Y?f+^bGEw{ds=>F++0Nlu#JPT8UO2r<R@
zI8(IyGE)K);=!0<C!mxu2R?-<dpdDmDZ&{JN=AFX=vuGA<qgAE!{d3uxz})JY}~o{
ziX&w;mfNNHir(0T%i**E-AAws!otVXMZIyXAa`%d#tVpPnEoo%ZN(Hlh5heYyB447
zaaWnXto9cmmtm;2KPn>KP8xQDa^3!xF!~Rd<7_PF+6{_Iv-gcP6sFQry^1|W<bI3H
zpgaB{z)9<(YscLo&6H|)b15Dd?c!yBLwD>^#6TvCd{s0FU|m4F#iM&dk~77Knf4-F
zYYqG?bE~}X?;-4k;QQFUiB%imCDdkS+Q&&$@Mp1%!g*LAoED|3$V&WD5yc#gheh3;
zj@j2SkeODaEW#~XdnZTxqiKR&M|nMrjT#GD?2Z|l3L429-U38ZC9IHRw8E+_M>%$6
z>xeXabbRE#>l-1iGx{ssZsJ6t4CMSeWze>}Xws7k*vN{*dZpxOg|$1!jbt-Hr;=QW
zlgc%Ms5`rX!5Gm3K{2!?r^|G=nq)VCJS<KX>92eGSsj@}&Jg6zhr$Ktb5x#qtVgIz
zCfQ`n-d4~pQj%G05;S=-9k;~iBFt|zyV~WX2KTD%fVuIw3F?k_1ehZgbPL&VJlFw3
zS+k+u%Oa^g-lAi4m%|-&obwbW8g&>bO^t6v)>=2&dN70z>n=dHMfo+>-vs7y#Ziu4
zti^ba9#E>LLzo(UFVT4(OKR>13h*G}bZg7?V}-f5F}XEPb`s($%^k0Zb_r#Sc~}+F
zCCuLH=yEIf6`&6D^a-6}1-c*BPt%ncBRI1PRXVsiK?wO{XosEf38LUW2hUQRmjA-t
z1FP(>qF^E}`D2BC!Y&rBp5nwQkWLk+x{{U5Hvr=7>9*SVqgUDzy9qV}x^TyqmdJ#e
zm#myUSQzT9?}U<@yo7Y?D+<LHrW_kig)mcbNs7{7y8!b<yIbT>#9C~y$VjJzR+(lF
z2{WhrTjM=Z?yy$$WfB-FK0=JeO&{yicEU2plY2sK+m5YAWHe|iHWXI~BXuufZ`VIl
zrMfZv=Bd8@aug1^mDp{6C>H>shHi!9F+r*{{wUv2*!4umA9q5Z34rHq&FRGdQ~h&o
zuq5SQ;lLurq!vY;=ROF?4A)jEZJ}-vmivHj_)3Qqv2Z@&>$FpZxp?VTm^;Yx1?B$C
z>c!FH&?1hkv2^lyUXU}SfBBy1iNnAc7d7LIiw-a1Fr<8dd{r3i<APY@UX+W-5}@tq
zDMu9F(4Fv3hpPF=Bf;Er(mKd!hYNcMrMHOhaju9w!r4<^BN7I43l}#ZMc5V8*-D4q
zZw0xrxzNIXL49_cqXAtb+<|k@rm?3V1LkBKMvjTU9b0_Iez>WUIB~5&YAx$a%?4AC
zgLZyQ2D;*m2Lw53nE|IsW7zTjj2QqQtvIGS0b1h|e_MDbIuVM>C4=SzM5Nj@#_<Zf
zM4cEk-!9UbHL68vnF%KW)%F^!Ii#*)&3`hmS&jA<m%=ehNNsY%d4RpWY+pyK{X7(i
z9Eh(QPAR_53iKxlan8vP4l<JkPX(%RYgAN3QU^V?7l&m6jqLU~^fdl4KeUO2`@EPa
z%pW&$O*;SSkWQW_b9p1R65VG2!69~iRCDn&p<GQQ3+T72R#*lIhF4`4?4K0&64cXy
z7x^vEf_8;5;bi}t+Uq2tER9o8nNu*&ZycMLUhg}*{z+SBvR{Q8i=6{XCb%DrNeZje
zOF@LK|MjAs2FsW_*5c<Dhz?Bjhw(zFiA<`nz7^%(s8be~Asp`3|9kOKH~V}>#-2i8
zr>4{Rwf;9=hNNQp&-3^@4@jqi%b4?vL>js0V~?@F5TEF1N6tKIT~^4OFDSm2@lVof
zSouOQPXmlr%Cw3A^RrtL(|tu#4Z85u{&-VV9;My=O|41ICqBOjlu{$hrUfsC$bGPv
z`JgE-76K<nA>h4AVJFXqdu$wfiT|5K`5sr;|DisQlEI|~&3wnZK%(<FsB|=oUoQ$G
zX)ZR<0dlwjCmbUcc1IhJx^J^c^R1CNO0K8kUSTe04~DFa$CfKWoIdLR%&%M>3UZOq
zI%Ye0RT0Gw8msYVLC#7dtZPJ?p~;vkZm)lm9vR^N=FO{%Pf!kxm9K%U^$m9H3~`h}
z1X45lW1M|$5ua0Sd@Pc`P|l9w*AaFtTHF&$Lim>{CazJ~#b@R!#p|MLV^UKwi{hKY
zXgnU<;&nn1t_()r@wrGhx8@zh_3G>MUk~ak%J(|AF9OK28)wBgDC`1p<D`ex-w|TA
zvWhTU;)+~Qp1%Ahb-EEg@&@4Cy=7a}hO2{vZ!9oBoWiQG<V|4Yhd<?5sTSszv`1kV
zjK_M=9lkLC$+cPhPhskb&|_R|bTfp@BwuM79jn~}<hsfyBh^=3w->4bsk2GTtq|nf
z$c@v{3TrA)7DIRCf6A2NGlk8=Dgv!}Zi6&Kq=8ex!bXT&r3>4j_><_oQc0JI9?E5u
zK=LZFC;lXd)k1A|fV<=zQ1SGS@wXt#IG`ttuFQlU3yOzDIH#U|WU&2rI_98s=I#^W
zl*(-_X-oIEAeV$in7yX%ln<K*XvV;B1~iH=yz3;COv~EYj=#qQ(%5obA|ek61g`%P
z=`ygS_r!#Ip>vOAR8#&_h4=$0!Mrchtteq-SG;mRfSQ|Z=X;?3Cx$+KafkpHnGJJ(
zdH0M50cjkZYjHq)QIvaI(wEGp=3fbBO<18JW_yUB%h;jkAy@oILF1I4dtK2kVZ+FA
z@v=ylkV%L}d8G~@GlsYC2%yNEC1Rb237gM4C`f~I@ki<jBnPtJ34ohu)%L~gk3yJ3
z7*VJfP7{(VSwSvLtZIBMkUV<OBYf*I!s?A|nmS|Ge?sNyuF`=d9r@Fb!d%5Zy#_GL
z<Ng{A)>uVh|A*FLZ;Z?TVaFG9Kf!;L4AYQOw1`kkl%~YwTA1}Po&?WLhEp8|o`5{Q
z&=k?7^S=bSRwb#5Rb$3~J2j4uv4g^7gZKPu?41d2(7IztgufeQozq+ENYd`F#_d9=
z{v<qAw8S%_T%9hN3@`i?v<uUM@JtN7Ly$Q{;pFZPpRmGQ3G7msW}X*N>zCDN`41$O
zZ@^0JJca#zm6vijsQ1PF!kjHznQAmV4V}~D@r~BlUw|XvWN>6WE7~<fMk)=!UxbjO
zZUQOMjL#Gik~+a)OsFH+rAt?DPa*W;@=!eWyFdFI^%~Ni&+=!US(pR891jR|w!9gt
z!~MlV@=U;@uGYF+P%i6)hEegdXjfQvxTynf_gwKA`%-Ba#NVC=a-F-lokcse4q!Kf
zR^rw#6dyIUqU^%{;4ML!57=-sBcA;t2&`uB7LUK|aKpH<@ruGOs4QBT@Z!~1fLui!
zsWBLJYc=DmfG!4ZSdp@=E`Sb#??@y+&7#!VO1Qq9UXRNJL+@YYwc_LEq(3xFfO)~*
zjTe>bH1mB|+#?#^&<U_ugrPLu<k!Kd0ng&a3ksWe=vOpR#;(}s4KRw(t7&<WHz8bH
z+#l1ZZwYXYat0rNe#^0YIM>JeBAgsnfgBc5MVKtiEJR`|LEW|WSbF1v6H#w$`8Kig
zsFX8l65Hq<r`#>MYFR|?@c1e@_<u#B{XNMO+)F~7wH}sCV$glx1@rR2P~t>Q2kMQV
zg;GCVj-!fWf-A0;I7Ebo5gvkH&0%&5amWYY&Rk-_n?9_QSX4hDvNo8|D2@X@0&u&l
z)%6;9Uz}YBd%AnFg!C4H{<JsQs=Sbanm%K^E5LkVuG0Dz(*;rfPIZy3KZfvz8qp9J
zY}$?@IXP9hxt=|>UM+-%OAh1MF<h7FBt~aHIchrpQ~46xxcwC-i-w7#M#j#cm_YY}
zJ=CX8rFIm#^)r9fQ*MqYzaZ?|=y_D$LH!cIj1aRq43f{00$sBHp5)5$8X;ycvdnlv
zBm;uAfibq(SN>D$6HRolNOwPO!dPR!hIXF#;pbBH6#<UGKB+h6`39P5$pP?M3VXHX
zI+$Y{yKU0kNw7<#M{g-(laSml+{;v>@4Euv7{gAY|NA1T+Lnw{Ph26`y-<(XrYrV0
zh2@zp+k=*3rXK*^uE}<Q{yyiAMLZf&*NY6+($9b{n*?<X&qsvhJrDbFR9<8){O#{i
z4sji6ao*z>;!=4GuHrm}&E;MW#QXgUnR*eoIQNV2e>}z%kL3T-6!@eF^8x`89=$&n
zK>J9V*&5&1|HdOOGK)QCSi4p}-sphIoZawbBQ)1T3rabj6+*h6cbIDzpAn$e)w`SY
zHui>s{FN<kW2br~?r@S`(|0ow$vp2VC9P-7LuPj3l#xyKgc0#g9pL_r_2;!&h~X+X
zAqM+IWa{x_Cd7)f7Aa!lWJ%vrvpIerE}-Z$TZuGR(y1t}&Hs}lo70$Qc1MTo#^2A5
zb&|KL<5H2D^V5TQtIq+I=Zgw@w;a(qy9vylGPymD2ZT^WJ?0&6ig2aN6%<C8*<#$B
z1txFtsq;+`=42$rpyWRY@c=<Ls2bgKL3>azZKmV6MBAmgf!!LVuEFG${Z2uS$?V>*
zH)DE8;6H-RNohenEz-5g4rCcn!{z~WF<7S<*>Q&uf2Ha1ZIS*82l33{S`?b+1*H2*
zjjmf^7rWHPi2PUnuQH`OrYr2m=t|Z>@BRTG7v3+u#N_#$7IOQzMq$S<S@<Qu+)OvN
z;?uJ8gTZdhk-B1|OmGc`8qI>m=X9>|BVr{1{v0;~99=2<F~VHG9#lFz7J@Vb(&H{@
zn+kCRo_LK}7lx#+Wmq^$VW-E-f}Qx(sl@ui%pgf=7(+;y9v0}i0Lx9DZcnd93=b#O
z&c*x-In0$p?7s-O2X1vr93jdjVrwBEn@cSUQtMN=M-UgLELOzov!Lf{A&vz%G#GID
z^hdB<jvg&R9K|ZJ{NkWliKm~*f$o)D7aS?vP030-GQJa?OQJ1GH6|@lAbn^GGZP$C
zN4SdVg~8<|iQ_h|4&ampXw#*D@<7r=UeOyR=L>XslbBB4sApQbz}UptrgBwbE^RuB
zSzug^&6WXoXBlFPh4Q`tCs--7C6o;9WnnH11`#afE%Aq81=P-+lAx6YIVre7ukR7y
znwL5-FR!5B1VT6H=6q99*#9Cg)Q<9$<v`3iSjj9PnjLm~N8GqPLDwChaojsA!<kk9
z^oXU?NGm(-j!*s*u*=cM9YHCk3!ty#TBQ`*t!P35l=!=dTn4_yrsnpQK%6F`Z!2A}
zop?sDBeq~`#VKr&l|jrWIa*GV*Pn!$Pde(VjI*p#M3a&+`aL{$EUdPx!I4vUmoC1l
z@qlo+(~h3OiwaX_Bo|%rlL!x{QX`K#tqN&AX+nqz!=YVaY5sM`T>^3&$`(x$-D$$i
zK{iq)G=%6<!$uS-p~Kl8Yl&8_>RMAXp<Qh3x*Gpb1ihO;YLi7$RAe}DjKZ!p;_&46
z?7keEZ4DzlJ{Rc%%A;pj3|}39vD&EDmSbB{P5@yKVwkCMtsoD%)<K?zu<2Do9Wi<0
zq9A`*gNSYz`oI*#9v2ItXb9%xq>)a!%6&WuOZ}oe$`DHzhJWeEaQbK<vx^H01~K=v
ze;!jrlfzbgBN%cth9lY!Y=%=b7={k*xL9Uvkq)y25^#po1;X4`&EoKp<DlL48p8v0
z!z9MuZ#=l`!HGe4cTWmQJ%DSj`6fW*-lkoglnl2Omgiv99WJa#;W$?5#jiokIguz@
ziRI9X2cV$fOe6m+v}W<~BwUCbEYh5nkV8v!`&vM50!BhBRw8Qn8-yxFv|f2Ma&0I#
zEZT7FiAM==I_aHlg#6pAQ=}tho$PM$JfZ#)ePCn-cdZMMYbf6Z8a#*mxk!zzAcqqy
zNd=L=E?RLxxgM0ci*E>|#-de2%Kw|fPLLyO74O(xvBdgd=CY2l2w_KW00Ng$6s*Kp
z5&oohF&rNXBSX&KB>GjRH~|p9x*`ANMGhwZvrfwLa@~#iH}e#oL6oB62tlrOQ-347
z!g~d{0^GWG#Zem<G?$&FxVTPtmCF5bTb*WkVhg!Rkq0AMH~VpgAPUS43zu<kh@yUI
z)pf;Vn?k@I$@srj*rltsv&*J!+cpC-cW^MTTflLfgA|Q|uWDxLi-oxks4Osvek8y%
z3;m1<LR*T<w<y4x{Tiw+0-4ZT+oNeqg6iFxhQAQuY;*&f5?F03FqcoxEZ7so4uW77
zr`hC+Xizk1pgAx#CT|VlVxcTdQyn3|jngo4bo{6OKeG}F9MAn8U`>q0x8Vcl*euts
z2xFGp);X{=@=US_=P<zCWHoxX1Mpau<sKI5@}SZZmkOs`?UK;GBSO;={J(#D!ftI`
zj&Um~+9^^bW}Sm}05BssVzMZ(^qem&GeXX|DAqkY77(|0>AvJqL9{y48hfGlMUiPk
z!?;*&C&F$Xj@a4YcsGI0nO!;~=dU{hxa~F0IP9G+#NXLaLsffGAlZybPE5+XKv7Yx
zyzw^?PLll_%iK!>Jb_4?jG0|$$ftG%H4oB5q@UL5PM%YUd;Y@zUVwEMidd1ucyFA!
z8*uKf60^`<J9Y;mcP{99<BUBZoI9duwz1RdKoiE;INaZp7_JZ6;^=W|eC@VZJz1u`
zR!(>cay&gb!$iYGmp~Yb=r5-08<&tV$99kgpg2DWcE#jVCWh?|4Xes&r@4tJm!D%q
z61MIj$c?S_KV8I37e+peff&0FVHX5b5hiV>*eDBhJ0WP%-f^XUL6lFLGM8#BJ}T)o
z3D(Khct)Uel8c07Q2SdwB2M_Ehs$iZY!eallENwSRgo?<ZgaF{h;Pe4<c|n68<xya
z0inn(_|P7%uz5h0`eTG>Cy-tJ#(m92lxB&?e-iCl*50?lS8fZSyAI<jY39*_r?-N-
z?owo>`HpS_$xR{0JuNKpwAMp{|BtD&4wI|M+Bhyjg5&_f7S}*B@gTd{q6;kU(mmZH
znVIg%bWbJ|ba8iCZ1Lb5U~!kl-CY(4`yq?VV$1h?>%D!O=jlJ{yj{2I)G0r8>J-^B
zZ<X*~qVedO$v{Yn;RS{8EK%kYFR1vV=z`9e(6#t9bZ*uSNc)mE;`p-&=iOImE1+Pl
z3CUVqGCX}!fQ5%CL>$(We<3q`+UONREDb#lphk0-ASc2wi6-Hx=2sj{tKI%0q%k;R
zP^5$A9V>ZCw45|E6$CN1wlkS4NR5nc85K7+WAX~G)&=pTAPX(|WI;UY+OG(92{p(d
zvkj+!m^q!@jfme1u-G{A$1PhQG$q!ej)adE2@93B_(DN{%E}l`_plm(%g#;G<eA_*
zL6%pM{nQ$DlO62jk<4x)omZdCc&CVVicI|sBp(UT;G;{!=T9|3-{++i5ti8WPOKM-
zI3T!fAjb+!-l<KjIt|P`p`OMjo@LQs4oxm=`&WzG1<`Wk=!sV{MLF%Z?m~MqG@a)F
zFgRfmI4-7%FyAF<<z|R>)8gh3>sVF6-<w}di2rMT!Cl|(_)fI5s1>*&SHTo?(F4I;
zIT$7BY=5*6GR1NVlLrx5%hC)?<vrhoCYRe-?F`0M2NBC6WX;J4J?dZp8cG*Rn@)u(
zW^3Od3zZv0Ilo@+fg_n$gk-hF<>&WAI>(+)COWnuTO9&qzI9`S3pe#x@lY^w;4!#3
zM`35rd_c?oM1UJ2^Ka^_YKZIp1vqPE7=g!gheKx#6O(Ir={N$!nalANmelc?AdeX7
zz#K&uqvhAlh)l2;a`1?#Ag9I|sHX9WM}m-_{84FRe`ZEAmG)|Vv;9&0-(XX>0KHrL
zS#Tx+ZBWnyEpRlDWmIBCzEG1a_P54gj^STw48>9{4i}LL1Jm%qSngN=w-X$Gvj&&h
z$wmVCgeU&GqD_Q3FQa*+mJSf&M0j-#1|G}ev-v2*xa0WO^-)A^p5F6*UYMC?_O0mo
ztrrBTAtA>&`gjPJfHi--C6fQ7_K7u5fN0!-L2<~mXA8}f^@lTJW5ihwLOoTOGs1d+
zfvgl43Ua-+mAU0U;iR0VQ&eJ8(X^EohOG-uCTwXit}_A11bOdM04)G+;Z->wxKxO9
zK|78^61D(>T!Ib^6!$+ZkC7fyq8!%=%6h02UN4AF`A&>mPbZvBJPqfcgK_H_p!5w|
z#3~cw>x{A;?TcsfU&bk`Uz>w1dKO@#3s-pJg*j(sVdfs4ic~GfG-Gkzj~#%{Ergl5
zsq*gobCc#4V>uQ(hyTsW%41^BbD>~eJ}+k|?1F3e{)|YoKz*pCmtu$Wz}#MHNpo=7
z`8iNePnq8*fC{fliT<JpOPAxyw0r)n`GJ}-@6umD9Oue*IvyA4LUQ6OYoDitxIhD)
zJahDdXbPZ*k=tYF-|~OuN+mWHZLZOr>p2|W5xo%160Wi5(OtB1%q@_T_YGF!uL@_Y
zfXt!QbOzazE&^8L#l|8o7GXB%Tqu-8=PL}3ebbV1tBXOi4u#*xdh8`YontCJ<(7<~
zU>y95$onf3rdsF+0$u#<O6fJ1fKXi9>l}M&{t?3?D|e}VP7!F<l2i9zL{qe0q}P5K
zVT+eLH1U;4bB%@}`9Ma9`&<s1ag9EOH?9RZMJWa2^cm1*iw2KJuDC;p<;)^UrxCUI
zpo!rb3&#`8G8<k&Ja}nkIJ;J1H;10WwDk0K`AQ(?tVLvZyez<R5HsrC{&?dmAQwSq
z=5t*GiA0l1bC?K{A4JsaDeO<#P05o~oF*vaQWnB`zv=pG^JtW<Dn}ka2_i{W)%7^^
zIta5gm==Rq3UDIa1JE__zWCtpVAKlN*w{|}Ta?AanoGv<UkOnaaCY+iOv2{BgUq(O
z9+J$Y8;^<zf5|O66nru74InN>!}Hn3Hv*BE4o)st*ojpr;8!C3Jv%c^Mw|QtB<ldv
zlG3FXJKhZDrskEFX32d8`ctVgIN<-A5H}89ou<E1)wP)K7EsFr(?jmT>><F712g5`
zxLq`|pmOM*!)}E#|M<n031u812>#iP#k&d<nUS#47pva}MQLhWP^3?tF47EO4BmL<
z;cKC0fDsIvp8f6sFauq30oZ#dbXLoLdXJ$|m`jKAQk`m4?gGg=EbDU_e$XZFX`<3K
zQoEAPeRt<6@-m>!`WC+@PqDkePNq8^65@Q>+F<;J_=n#MM(tv*TZs!qQ1E(ZSKN|?
z%iu@fmnVgfElpip-49}^X^7H1w7no#TGFSbBl}$hl43h1f`>G}85q>#bnUU^19>vN
z%=og%*+39qpl*w6vvAA!QSs4(dGfTJPQ9Ws7<2s-(5(vtli0aQ)_5H84w2?49RV<c
z)MM9&0JHYWU@LRZB_0OKS{9ojDt3$zN75x#74Ks~S#{*P_Za>Ns2Qjg+G2f?{w}FU
z7{R&(c@e>5M%7|x<7J1p^#?5r&sfaSOFasebqv|ta;Kw-*oHR#DFo59jGhorDcopN
z%;_JAcHN?h)m<Aj;E#ck1FWEXxhtb+$vDk_Q&fXfRC_G@ei7s(c@3{SmV6wVra?RF
zk9|Zqerkve)$>0IWR|;eJ;^Au=~H<mR-d@gC`D0FLu}GXYb_a9G*L$5$#gVrS-%%*
z26QGC>plbRjwJnPIZhYg3T>^*TdJm@Gn=twM~NCn0Ak|yP%Qi`VKwjM_*_I*2Dt`;
zkt3f2au%{uCeP;q%)?~aknAYhg+zmb4ggar5Kq3m`JipNbO~{y;H*MWw8qsh<{wI#
zL04PsB@pL@RU1I3fGq6}J)|AqWl^MTN(!gvU(Qoaqt)Om`6tW`eZ1~_gb)k32mh10
z(uw1M8-=<#Os;nLI{PZGf?0a>1lgXT0C`KGdB8|VBg}}`@&ri^Gt9IB-$-a0C)j*m
zD#9fhK+#o;F9oP&P8c6Ay^(*9DTUO0QlsDYX8wr`AG)IaRt})DKzls&Z2&ble67U4
zGJ?sg94EZPKZq9XRJFgoBmO4Z1!;Iys>Fi=U62yKMyROe{srQpyR{fKQT|Ae7Q4mW
zAkZJtTBys|?Y$h4_iZ>|?tKs!c3i5Hl?7PhT%>Pn!%UU>KVPT>gndNJ_5p;4g~`mQ
zONrz^=I_qfLIkqn95|j(IBStSda9>c-W6m%d4&f3rL43LnFZ*^Rnon`k3~EEcC??f
ze+bza*lDBE{4XcWMU!V?M!rXdIAt9jqLko_=Q)8+xx2kskLR18u^^Iv@?rl+44Btx
zY0MO1Vb~k*>rH?=bo8i0KgwgsYN0QF_c4H(L(##)lVm0eYp_BAQ`>%8pbJ%KYmF;E
zh0I1Mu9L{pY|4M~q*xr<Z2CfBE+@xJ=n7)<&p;XytYprhb`(fsV5Fi2?beK<uE182
zu70*aXGk*XnaX<vnWa8fcH=*XbWzedGbSr>fiTs(Ovv~C0wNpjP$D2<ouM2nlqjf6
zCdR!Y%r6#H(sz&e62zI*v83o038!j~c<T5Hs<9U@^kezMBC`|+3eFm??y2DiM~F*}
zW)rs;iuR;Ha>hapjrZ3kIM6P{>EA#&UB3b9yNI_4b_r3>X^A%&L%s#GT)54gPQEu0
z)My5M+=W^5I{=qKF3C0$Y4(c!B@DOYZ6R=?J5#ab_xa!LdKnWl<^O`19qB)joXQy!
z1)H68+fLQAi!hXKBU@ti9|+QwSd_%t3Y$yLI8ztcMUamxSntS-9N#nMCEVQU)RI%B
ziefq+7PY(qLbYu5V~(+A?oaELpPN{EY^s3ZNB@+^ODj;uytr7{Z)QCjFEJ<moX70I
zkxKHxwXHC-?VNEtB`ApsgUvlMg4OrvxKD(oJef1&ng4@y-<>SY@`JxZyA(D20+%(K
zuQJ<)jr2SE{i5zVf-LJcEfteaqZZo<&gv9{gL3TN1mtRkqg(-5Z{W`KVzWb15%sn>
zV2=D-+%DoKY?c6O2G`y(-<<iM=>h3jZ7zUp%#jbvPSKVY8ewLz>8j^Lgt|0Lyj;Db
znanj07@}#FEo)tbs}c9p%zSa05OUSye2dOU*n(wlr6-nRmmxsTCp{>K*&40ky9N9a
z57x&b3lb)My`&yHECgXTIgYExaZQ?yMIkN{O>X#?i*FS+FBLfeWzoMdh#Q}5H99N(
zT>2)vr-fV0NCR84r56EkIi(?BTxOvo4AB-R;|Pf$EHNvV<8l%1<m_0vl%Pvx4+zX!
zK7iD%8!QF{YX!XbZ>6wVW8o}Qy$&JHk=2*nDE7sHi-S4kbZ;}+mfbFpytxaG&Ezl$
zGtL=Htv?PHKqj0p#J)k;nb1{`c2tG%FUrzbU<tw$2ls`SDd>pBDl5dk*k?%)#LT>f
zD-wpOjAosx?Ni(@+WF%8j(RwKDFC=Cm2pddfe5$BG3*l7S~@3rxHqPYbW>;Ksvz!i
ziAnyHCY*WnkVQ)@zf2w*qdn&FLb|GQreI3K8EO2eutmpY=0ovsmIZ4FMt3geTMob#
zqN{dl%ohk^Q0JVCwlU}Oc`C_#mo;(h^1J+_qy<6`e3T$3j7=kEE_Vxn6P&xn?^Yme
zLCWbp*^X}_JdCZA@q>go0WAS|ZDmD(Y?7#OH8j<7MW9pCZ3(951Dl9uDA|&&L>zKv
z2NQo)*fQV%3(Fds)W^c86I`2fr)3ogvMu&wGNICMw<<uEEgM!E3}BDItjVxSq<@|%
z#O(U+#ODIshtdWv^YI(~tApG3B*tT`1n_N!YKbxHF#Q9cofylvt?`?$rN)-KQ+Qo?
zO>nOZd$0@4DB0d(mbVuF(HME9Aa$f6YeTy#@qn6kDTfF$myC5B5<Miq;{?yc^hC=D
zXn5eNdJHI>^-UQXQ%xTVa*NgbMGLM2*%*|HsX0(l9YW3HKx-_zE)>#5IX@WxYY?&?
z9y*eLoIaPE%l$0<CkV1!HSn-pzkHMvL)TMjjVUcqSrg_^4$VL*mKdF5a^#WLz1InY
z1?}W2@m@0qZni2s(a{|nk0FL7!8pRQbU=W!W*Ac?AK#3^#+niA9RY3%x_zcY|5_Br
z0$K*?LNh4Z*qz6L!a%J=^NzbRO54|BJTjjDGPx$T;9yLh0EAqV*QvocP$ad&@;p`B
zQWK%oci2#TuCPUruI*~P9yQ7FNxx;{m<iD>+C0(9wH(nFb-VRIUF1@A5VlH9KzI5F
zqqKgWPCA<#x&gHF!uZR}r+b7^pS6Be`iE}_;f%7;SPk+2gl3%*LnC_CT)zjQe2njs
z?4=^rBUx^Z{zIMw*9p6OF(#x$PYZRs>|peZg*Jj#g>oi`UR{*`O;3ogE7}#)%1S}z
zPTg^rK(|#HSwFEcBz$wZ607n}^8b1nh(BwR18mjT*_3}RC|36@e)biRHM?#`xFS1}
zH8ull%#qTMY9~b#zfU;%V3<*gwf~rZ$XLWgOncp4Sc6&Cb&Rp)=oiQ=!)EVX1<iNM
zgi*2l=8&ES(1US6&eor7))(B!9=E_clo7szaI=cim-Q$)?ko)9(S?{m&dI2B1`{(y
znHzSz=*p!px?Z4LC=*AT1YQ>8I^|lVG&JjP0b)Mob-P`pzmaoG@iJ48zhPGG;!tzl
zEkVq+cGtOwlySn%HAnnxOR<AkY%4GXiHLaNM1*S`YjXTOHfbz0iZMkrEvj#@KW=OO
zM{k%IkBN45Y6De|$F~MB7qV7Dr0O5F_@8h`Z0#OoQe9>n0Id>>vA=?3z{SV7R$(|}
zV_J`4e}Zr!bqfId>I;OxE5}~R2Kw)!E#d;E1@6*|ZVQ-oM%gaMu>xE|u6(z|8%-K#
zS9n_4j{oNyGW)90su0<;tZs=5L}WdXv9D%d97}Ew>JnzpI{Fy&7~!s@LU(c!5<C1k
zkIMOVXLsXu#YK0>qq18~6Vz})W|0MFA+{BbOv?4?v5UfP4MpaWy+x+VLyUbQT)U`2
z=$p(W(}g)Lc0Bm8Y{t>ZpDy>ly<;AShunxGnnc-K)?&S#AS^&R$LAjW&H%0m%vb6i
zHGE2em?z!Q{ACwNvr)ysIaT?pyXH}lT%0b_T(N4Y#3iEL`82yVZW=AP8?bAXduNRJ
z@`<#UKr@6H61(f3-Sb%3Ot*3GoIbJV9)OgArw96Ch6uBqUUjd<U-tCha+eXui}2rQ
zU(n>n<w9H{ZnMPIB3=3D1W_p_^Ze0!{pPcx=BOz`Qbkqcc7@G5due&mqAzc=cMj?v
ztSqi-@u@(|3*|GD9SX}e_5m|Xa__^Z-9nlN%}nf>E1<{pN0w2tEY`^Xu|RWdhmc4Q
zW8M9JgQk~!bcy+4p$uCr;+UHj-ye!H$obCs3L``g>bv455te&V=KgU@7L#Lvcu`^V
zi_QlnORQ1=$;K(vIa-?TE64>+&ZNX>MVLRVaNx!SGPDR{))_UV!jXaH?5#N}4|v>^
z(p0;kP^YLWNjK*g6Xv$XO0zGfigpg%lGCB^uR<sm3f8`Op#(vB`#B7FLt!V!SsPO*
z^Uu-c{Bx>Nl{lmg#EpZeMB`5)NsWpt#T6nfBo=;LLb^wQYe)KO?W1ZjrXA2MA!cpH
zdWAS=94C}@<2Gu-;0z^PjOifkJW)r)S0Wu<rZdUN?YmAes*x)fth+Xu43+gTlqGSX
zXfx*3_!$DsgO(iduyq%RHk2IdD(F9ZG<gih>q1~gN51QJ=l{~QyJ*5k3CZ~Bs178*
zd1niQ<sJ+TIY4<|q=!moR`j9rYEEMF_JCVJTE8<bk=rnV&YxGKwRsp?0r3bSIpRf=
zOH=h2QU!J^lQ}mVm{?3uCdx9ZR~h*%F3fGp4-)J2DOPC4%#P-2(cQ$X0w(XHdik0f
zuH!Ky?h)Y}S$eedepkelJV#~$S|`O?QkY9n>*wNlB?g3$ikfts()=5XDP6#OyNMLb
zsb%iimnWgu?qTF$Az5kB3&?)tM?qPy>_?*&zY55z0;71YuHn+44o2m1p*VW`2{)=1
zwHl+A5H}TRlj9Z9u939OPlx!YP6ecKup7EbLC3FYn!axU+AI{9n0Zcp8i3iPbtbLL
zN`m~2mZq_aXmiPBA3O3GEzIfQ0)^q{X8~GIF;LXv`RPz*oHut^aK+jOfMly29Ltm|
z12i5Q@@~Q%lLHWS^1lipCEb~;#Q7Q3TN}g#W$6R?&#Xwz+sjB9txb$~w3(6+(<uk#
z@#JqfO>3_Tb52vaR>p>8g+oBx(pVBU4EtAZqP2e9KiM#14WSlFI^TOvwA;9BgS*Q%
z+<d?JG3F(7owWWv`%of#MCj?59+Un8Nfp=WJ$EQfX&I)`oJ~6nDw9TX#T*w4X*6MO
zsAC+u_Tf3irRlU1-A$0Iqh6iHDgNLQ02UOtcq-Vfw8uI}f{|^ND|zD-c5$>DitV%i
zdN9?yQeku3GHOD6EYfNBRp~?VZ5Fr6lfR1`#h=ax#c%<)0z9aDG#L4`Od1vYE9?UG
zXl}&6gAlqF%Jb1N=~xJ}k@g_FH))os<KiySE<G+gIMw8dy5oRcX-SQXcP={Y^2dXk
z5BeX1XS&)zm=mv8@ozFHI^zX}G`@>BJ|U06lvIeTL^~C>6w}dM7Gkp#!CZVDbzp{l
zQV!J0I4_7TPX@5O(7{Cgl>C48ZN(}cw*bsB8^UB5!eB7(X`to?oeV7{mJ#B@7SLA+
z8X?G;<8GGBs{)W7k1G}83K3?PzRPuN)>6Bj4u<f%c~I#hg&Xr3YL)aV-zx&mt&Ze7
z&TxE;45IZ+!WLLj_B5x6R@>C16AzpP<rc3;*E?ZQhn_Rf1~!ZGTwjWF&IKR?PDyoK
zG()7B<oJQ^K(Y2d&r#T$j~O>AZWE~n!)3$w6eiD>bTl{Wd?+`Tu~E1H!tBvFCPkNM
z=ggZ-uu>7=@)qbRyxWIS$^F?^y_^_Gr_uShJQdVW%ypt0t$kt)zYx;w4$vdhC5nTE
zxdI!i1D1WK359F<L%&2}vw~9}=DM#h0m!6`A;V81DU}8|uIyY2<$@%o%Fd#*F_=MG
zHt{zIqTuw3fw<$cJicCdi8U{W&Kk1Mc&7+){!G;@^ZzTrENiA_64z#~Gy~Kb`4S@6
z{RKI#0zQWr6>q%)L|N<B!YmP4laQs_LRaQAJF?L+^eSjo9ghlE73M=O;+5hs5td_m
zE&)wIJR^+6vd5s}Ezz#8HqEQ)V0z7~0kdsdvN{*@ERt>(&ZO4X8<Vaf=(H)V9?RgG
z+O=So0QoU9J|w_p#k4eSI#;+3$Ym{YWrShl9wDx#B6_4ZM7p#X4R^;ve}`^}m4l3!
zV`dJtOlo1tc+B<r7nm1x#WpuUlOTpk*pbG5BAp<Go*o^dX$S1h@GyJ7D6_~36}}eW
zK#*CKuA2VyUm-{szlw48jf9;XvtwucSG2n%yJ#LkqEAw)H~#@_`S1b}nl5I^Z-r$E
zWBgAiT>GXxG2JAK6GT&exGv@$mzyDAlpb2?h)+fN<Nj_fVq=|KK%7N#ua|Z`!pt3d
zlyWS5D>Mn}3QVji!gXCO>hM=1^q&Q06=^f*O8h9yRiXvX9JfI_YerKU{H!YkfwVJv
z?V>PQpsMSSyYGN-7Rg(DlEs{N0uCRt7zJSf)KO%9K(_f^{FSv3B%nj~L+%DLL-cgc
z7wPB2?g1kYEoEbC5!qNSlO@DWZ@oZhCJo-CVJFSIlJ|nS9JOjO{wNZGB3*X>cZqVT
zSe45e&)Y)GJkF>v{hp4}ZH4>tXnKT?ZP_1$xJ9yx;@VRJoXvDh9Jk2+m#qU<MjYD+
za3Qh9Pm8k!@6Xd<-!pDRtRui_bm8OqpCa?REpuz96eAu0bKRnG8;p$}&g0lPjA3na
zVOdrkofx<c#!iB$tKPxB*jZuZ$izP;4ib@7s}|6eI7~>!1Ut1w9))y4ndYhX76BGe
zQiibM>x&ZvnyoJGCd6%zL7N3#Z`JEgTRaZrbak+Oph#z&lzr?NguoZqEC%C85zdxF
z2>Q}(PXJJY>C>xm?o)&<m_lhXR$d6^8DUNy+t>I+WYz}Rlxjp{3;DFuz*kzVuW(lL
z+}S`Udzg@{Q>%*qDLU)ajm;Msy`LpgL5uRmGkGF<yqkmdMV<xmsM*m^OUBAau9o6B
z;aPDRNEt^Ldk(}57uvd`F4DO(hL4ZGiuMOsP)mT<2tfjq+2&sVn}5qP^NNF?=U+=)
zr!m_8KP1R9<Vd`0FdlgUz;a9016VZ06)%E0AKcLMlwSNS#Gh3<ry=^eU&;aWowx;9
zMgTR1c~NJqEFvohUnerF`DvUNvkqNFF<`uek<LhkT}=Fi@)U*Wtey+~jMUQv@O^R}
zY_0X}+4r0r#z)Qg?O03}V*i(k?|}eym)4t{O2>6Vv%170Gwu?dbtUfqvi>N=EMd-?
ztLEKpvB4_<W}V#=`_yurEXb*2v&D3lrh;W(1*Jf&vtqKsSq%A_WtW1;9un$<H10}y
z_kzQHI_$Ih$C+SCqH8>{(rZ8&vy2z9iD)y+W~7_N6SjBFIL!CBxPARMaeCx)4oj6L
zh7&$CmlW~=VcBpth|-D0*kH{02B7O)cis4OT_GwF7A3nYY`L(9<1Xo6o6#|$tuY>c
zCd8>?=fRAs9$0)c2l3F^C6`6DnAHTi2#p3q+Q^>;G6o}1wmIdup)3(;vos$)BZvmZ
znNn|D_YQ>l!Q_s<bY~N&_salqlXvqNr~xR>eE$Nl1a+}e#uzb7SeA{nJrrZ*-$WzZ
zbUk#uFz1F28;gr4-Uo2)bZG*e^+BGFN+g~AkXb;MXQe!#`Ri&S83XJHlCEjgzroy6
z{QO7SS^OXr$>WkIX8*rD9VrVLB)%7tRUcEe9E0Hd;-fqo_9QZ{pDBbj*<f$Ht)SVJ
z;agnsF*NBHS--|7ge`(}PJ-xG6=X)4yXnuAaCKqs(M7KAY}$;X6H0Ws<=9ggHJI#N
z;$TrOKQ>P|l#Ra$$+*z%jPIK<n9nI&Ec_`kG=^|?8^?-(Np8x;g8w1xA}6I)T8thg
zP^Dlldq-iuq<Y52cb`ExqlzvptnfKBywlYCFhc*6NGH}lNH?kJ{PQ4zE;IW;dsi6#
z1(>V7*exTN^95K|_&~?g8c(o%+5C=;yvB&wQJBBu?RYxN2LfQAm&YThtItF_nIgv$
zy2motS3qW&+}YTQraWy7E<CCSfnfC-T`g`F<*%_kNVkzM{Tj$4OJ7;GQYfIa)u#SY
z&hq!)8rOdVY0lW!qCR@*TY#*V3Yc>8bnbT`<ct=UUM0d(VW6IxW{^*X`4g=T7}!4k
zF9;PuqmCngAnbOc#n<dVLAre>?~p##Sw^UfAR`D36k7>$PJS*GEPe)aPPzflwU{-9
zz#My59#9nFdPFs!Y@KWIy}(S6D0pJTFVIdm9df8G?<cI$oyScW5p(|!z!~A}TqlrA
z3UM+WI#^ptG#Mk<j#x#6)9fp!ORl`wD~viqRl!o{Xi=G*c|DmX_$z?rkSzgp%yHUm
zzi%XzZVgD6dRY_0QwRDv#>N|jkvQw*fns#b4&}sghs)aTXaN?dtk2a1PZwks<v}mm
z9c(^Fj+F-G-x?drLxj2HdM~_pFcz3IM_AO#Qs;MbfmqI(D=^Jq{@z%i7eZ{4nHbsA
zbMY|2PLIv9{KCGci10os=?uOQmC3crvFOor=V{}Ro7dX(PUZgd06LN6>$@KR5<;uO
zZkR1XtTnI6nk+}k(Js;_I_MMO&h`U1&D9^7V7TX}P2~ae0l6?bO4UNV9Ipw4M?6<h
zhBrlJO-SC77oR^*g+(ilGOF>?5FoemUMk@E1t6U$?MD+nTDv!!AR~&5fN59%U67b=
z2U1^a`%Hs;ej!j=7j7Z?V%~*Klyru(kqEk$Tu^VWu=(O>tvwDC?PX==51pZeRyF5P
zaOXjD$VRi+Ola03nBUVDZ#!XbNt|0ykcR|VU|8ic)DFf!7s<a%j?tLx<4IvIuY7VO
z?R|`06c8bDRg^x`FUlzuD$~*of2APjCR^8Xd?|p!O-H#pYB9nt9F8t{0z|Y6p+1Ly
z3$2)UTO3sFz&eAML^=~q8GYzH-xbs-ybN;u)Wk4QcpFef#0R3x2-on~I1USdEOR^<
z#6LvC*qBkP#<EKg&Ke^xhv61rQz4FxF<4t1xa4nQqzB1=Y$6;$NiDm^QpBN#rI?7<
zMZgW+8UI8-h_W!{vnRch8cPn(!LH5{R*up;t=j~1%Ziw1X=1uwdGE(-kClYEb7Fko
z!D@h26)s~0n{l=iqoXR?MI&SR6PkY+kZknkKw?pm78LGqxo4jMXNFA)n`P#RiHhXR
zwDj?N&$zS|Oo`L=s*OdsqNdVCHS=sIh~)_uSMx6Gs9Znmi!DVsDx2?KsZDoS4#;)N
zJirPz`3l-qs5(vZKDvDVl@zu*O!_}TmTRtDi2wY~AC>g*#D^<DAP*@+)eV<l5!&+5
zXhuufvx$`O$<5&Kj1Y1~8l7?RN)YCiqYXL&574d*MAN~HC=OHD_1x7Ok2fjw{U3`$
z(mtw-<5q#RU@J0x(<PTXR|T{5SwS)~?68{C;}DGUohgc(T5z{NY<0q}H{5T;?jrpq
zt8PZUmNfxfM~tZ1o8z1iujS9Uf2+Z150P2z)_B$y3+_0%iS=l7#)Wcxxi<0Kj<{XI
z`;#M}k%?Tc$Cz~>D6~uu(6Na!CsSBp_r<mA`rB;8ifczgyO#L657PwppPvXMjq21I
zJJi)iL3wy*cI7#1t&uevjME<$zn1vg&HuT@U+Sc(C$14i?u>z4dx5!cGk~6lfd=-#
zM~)_jn-&*0(hVB8?i!n;^gBlGv|UeVR@;27dA>&xDLLK)@vf8+ESL-NxQMJ7XXikt
zjstR0yQQ74OL70OK*y6dQ1(u9j|Xxy>_c_VN@uDNjYsI<zf#yA;JFN$4aT1)_-{%S
zb%ibPUe=?1@udJa4VrLlI4S=h178@}Qh+P7O_h1MXy=|;_RkdJW`V9p6Bg^4^}sAQ
z&0}aaigeVW>w_W=Gz;a}UIdkBAEx_k0A;qj{SsxoA&64->#2ba2|HI7db*l(rx5<-
zo@mVTdxwkFt{ASc`IT4nnEemXWMfaF!$p}__>!}LQXC}ABH>Om%c;|v0G7LSk8_#;
z2RZEe|1N;Rsn{0xD`?L2`XCa0O$ZXT^WS(wWFyO}H$NW<Gn1?*c}IRDXqO(d@U-)L
zR1oFUrP)y%6LubYS&5P5z$PM_5mao;Y(g9jJpF3N4MO*t=G~Nk%mB{0we}y37GX}Y
zv%=9B3v40CN#j~yoGD7OFLuT)3Y$5u<uM0gBlE``VuDZF*Ovsj_10KDwx`p^?kxaG
zl>USp&ih1p8^T#NXXveXS4~6N0$cJA*$uFaSyy3K2O7??BjWD@oPQk?z&2J7$881V
z%-AQlCl7`^g$Rh;nNnqKbJNz4ma&>x+%Gz7rab&1dzaYvPk9_>ol3kdI_n*Hc<76f
z+X6Tv1_>$6Nckv%Zroi37AGA{DV2CcC}qX-04aaJ9h7t9qzbQ%G^1JDgJnYR!5<EW
zr9zxMOI1X#^;rTPtpoKci=EYWAR2Xx(gcl@2xO%<*N5%sF#DUBq_98Z%t#HpCd8?;
z4@?_ACMbcHCi^+|0w}cv!MR*il;gvlpvVl1>5f=#7YH(A=!=6D_Lr#3SiD#8(gkEm
zREz26s7@7M+7;ANBnP&np9vt95)XlWt1z|1^$M7Z8M{G}n+EN;NrcCu7M518iFApu
z6V-lm@!dgOr&^H4A4RLt>%8@<Js>Rd9&|o5{ZbshCzyFg#gZ1ar#2xr{Atm@>RwKU
zy*4j3itr~j%m<}KJwN{+mG41&=h+n4Or?X0tA)V_T4}c9B3%4Zzn%e`vJZgE!7URy
zP@KIlNVX%vZKSQ?wZ!<R{$OC~=#7UJ^jDa%Vucc)3UL9Yc9%z)cz=JetblR=QtOIY
zO$c3TGSU4>5EURJ=^cuM%^nxx7>JK=1)%y-825I>C!(Bfnr`~yCm~r!Dsx+TlM*zA
zMXkmuM;XGYO~t81IbLl8aHC^o!R}WY#jV=`&1Jn4V`I9@?t+{TuYq@>fZM1e|1$Hk
zO8eMSpfhIzF2^OJGpVrfMYZ!>CkX5$E#+pD^S?Q!la6~AA(W2YJr~n{5@klv&Gt&|
zHlhp2ZJ5;~W+_Q|^!sj5%R1>(rTksK2Mj6JQ9!PwuoJ>LS?_cPt{VjT6BY!O&JykK
zCSkCTrDamXty_V13uo6Idx>;u2Ff)Zx_l$V;$zLgnuRr0%vW_P45KJ=$HZEqNd+yA
z-uT;IBob(qS@;Pe%$SV)xp)>U_X0Uh**&t{pfGy~OntH*{Zj~g`IkJaCo^WWpMAiY
ztmQ$Z7B32+c}^S|v)2idX&V=ZVsR0!CC)~BV>Qtvt_#DPD-5&kR5A;^<3%=_eljmp
z4`MlcP4*2=&0}akk({%xJ29rLD1>&Vr8@fx?xzT_wB-hC5Y2IXCoEgWatnoSqrc*?
zsd-d(Pnbnf>njg{Y0SgdWLK;;4GJFRcAMMz>xp!uuBAoLT>@M_7It{|6pfIR1||)x
zF*pchmTA4xLVPa7b<HVBtUn#n3~O7ThWd+yx$UJ(PIOj=n;V59Jxm_rx#m~6rc4*4
zVwD5<AJOAx3Gdk=h+O3Guc~lX4cb6oCK?Xth<fy)`#?yy1KKQF2+j0o3pC$4YykN}
zh+9yRZO&9$*pdeUsXhmIhV<Y(m8750qCZ*%x^VprXvy2abHZSgGsBh`aR`K2>(*Nw
z`yL97{4qLY`+LY=pj>fGvy|Jz>P3Rdwu{r5M-?`o((uFQ+d{GyfE)a@kXZR}e^F#r
zI8I?o-=jrOIVOp80*pV>r2IjM<$;fd_DVY8*|!<3q8oa(xL63~z|R<0Z!$uTsN;kq
z_{Y_k^b6d-C)qiUbo>%V5ivxB%9-31%n;><g}?t%BjRxZnOHC*h;Nz^TC3Q;$4N&y
zu~t3U^MMGLSpAgqIv$KV8Y~-#xEHEPq)iZFYZ=`VdmTgAIigg7yIK>FG8(&F0hWSR
zWK7Cb_-e-jIwx7u$bnZ}Doi6OtA=`9B?_q^%dYt3IEbv{VJY5N9^7(#o{C&Jv5l_9
zQ^L$IeOV)ayxWA@s!|hnMBfR-q6kv-|D>?LV9c(tYKgf{1ajBHGCh_&2@=j2AKQDP
zcuM|f1&z;Yr$Sl+?7X<sxVL~tdbA8`^;<y{rcxLz#1f}LGz21p16g#hE07rS1<Q+x
zCOW52AB=Vp=4F7%{=JM$7F4{rDw2X=Y#;NS;c$uchAQk#aHqyy8@03*1iFc0%*);`
z))Zo1_3W390<LX-iBgq4juiL52s9I%hw<PdB)Q@PFs7YF7|#1y9K>fLoF2!0X-how
zY@kNZWXU(=uh6dgw2ssP#RI}fuY&`pw-k1KPTdPJa1ONV-kw^0dHo^5PR6h5(bQ(0
z3&!*@eq;<kkDwW`vaS}J339^IItyHH5bfT@+@W(bsWm2C0IGIJe~(rXPKEV6akv@#
zHz3QfRsJ9<@qmydjB@<Y{I?4?@EB8$yf9BfuQh2J870VdQz#|h2BbURMS$jpd(=9x
zmHuJ_p>RW^=hoWA`FB#RY82z(O`zo<g))nhS{x*d+R~vGV);a*3og^6SoRX=#wgLk
z`NLe7IzH{YhKGsWM7q>GM1*UAN*peTBxIPYAz+4RdIU}iIBVSKGXKl>3`#w5v}nty
z*w?Q!!|R2($Vn$mb1cQ&mp4D_XC1M@3~2M!Tj=fQE~5Z5qP3Qm%44qpa&bw*4rWdk
z1~WYP3TvBP3FWMKSwIKDj3!lqS!=;MGF^x{S{U_KZ7;-e3cGzz;t8!MMbeP=SwH@*
zptanrG-(9ws+dyjcon!))N7ULF`So#x!gJ#pd0=w#I>4fLN%w(b2TVA7yG;79fcbM
zsC-%%t^sguVR6U9cGp5@<xzF&UTo|l&?6GQ7q1iLE@Ee_TG<W8>em6gP?O<!FOlY0
zXW_lNCU}CdEO$Hx55_qHGLdUVOY^z18E0~JkR|v7O~7mV-oNva%gyE(Rc#z2q*0Za
zh{nePTnd&2G9~B4cBPqs<_K5ua{1O5lbe_ng=u;JtT2nDi(56Z&h^k#Pjzx!ub`7i
zix=&>UlQi%13KuPcmuSVL6eV2H_HI8nrWW53ZO!2oCLh9uts8)ea9P{N$EPzfuf06
z=<1Jk|KW)A)<Wzf0$HQT)#~v+kuEN$rhRqk2jg{tmMvo9P6}43HvyToQnG6q5Rf%1
z?A3YQw-omXa}#8xC2xnGGv2xx98U4()gJTRnt#}Mx+O}&+yDl!Ke}I}Gm}=l7GK?#
z1096{jtSdu&jD=CxvaO(9ROyh#%RI;Pc_<vA%i|%Wj*Omhr1a2TkdjrDmJg%-%XgD
z`wGc3{k|fdGmZ>crJpOnEe0F6KC*teiO35VO>u#Hh~vC@-jbmWQJgQ#BGe@gSsq+0
zNL7Lg;d+IUh;-O-s|e?zQ$xne?}a!;rq8m>=Hl>s!7QVeQIq0!ky(M+Z>BeXUdf{1
za!%Fp^L<2d^Fo<ck7Ms|eu8xaH|5%6s|Ubf1hqpUCTD~e#BsL>i-N;>RGA+=n17xW
zCfbD_^G`6!B-6|H6m~J>SDEp9#fL!L-FY-DoviB=X=TCY3-90P@E(Sym$XieBNepR
z(w2f5{WL-5V=~(P?zrL+0JSFhn7&1XrGjoLUK8o0b*(SPJqq2Z8r>#DEEfyo6PBGg
zw2Ci8@=4O4@%MI*`ETY33^w9Gk#IzV#;z?LUA!a6iL$ZeD3RfQmN54t<_#7ta*T2I
z<A5GiHJRWaUn<90LXjC~V)2>6l!Z6YCdIfXAj~}Hah3R^=uDv8jb+I*LkO%F%W4+!
zsz{4Q*Vs`3JPBa&vFBxN&vhL^X3Q6nG&wEzR1UJOWc*N5*hC<F@rY#96;6Dy%Rc>S
zh^$gr7HF-!@H0SJ&br+diwQ8VZJp^vc4;Bb*t3zuwAizGG_6x`ka~y^3zIRrj^n6u
zto|GroQ%V)epiKE!KjR4^z)D~CNtsa6G6k7G%^->fuQq8?<nQlT0)4ad%3a9i}~Md
z`(*_Ej1ZSV_r+?l<janaU!a~?TZCq6HmKe4v?vRJolP=3TkMtmbBu$K^LqlahR8}h
zUWLrgoc+v;wuiMC@)|f?F^BO^sR&m%3$C7+lKsg=%|e{i{1cBOg;@V}A~))mLa>E6
zPms%y+GgA@z*3OEJx*x<EyQe<*#nf`$iMJCrT++UEm^%I1!DJ`fS$CcbjBQS<=-&T
zQPfcaoD0e%t|rIpLR=86CCc%hXeThWKfY?lXOp1?)#`8azjNi0=ssI{ZYR*arB>kF
zNCK-yLY<iw0<6Avc!zjdi{XA5+aw%i3$t)IsZM5av%i}|GCG!R_m+Y@9;t(oOHx|)
z3(ne3xk9^)6NS(KsqNV6UwJx=qR9;XPC;g$eqFQ~_)YIM<0X|#JrYD^>!M)Xq5ny<
zi-B?YEW(XOpBD7(@ue_xfuCrp$@l*-N0PM-?wp1AyV@W}_Bhvw;r|9B-Ih@;algWu
z7?=Y3STU^m5fD6-3LV`sUPRU>z0qeb`jbF6kuHo(?-1!m)yZ`RY#ju+a(TlG8<S@I
zmX>j``p5a-I#G@@MK^emm&$saBgjSL5*OoO9Qp}JCIGpG)O7XTr+F;KeqFa);XgT2
zkRy>&Ocdf8WF0o0COwtOE{+i_*|H_RUt!uF9yidqKau^z;Yj@W8UMJTcxH{RFCbme
z$!I*<qV-DvM3t^sek{TqR@3b)WO?dW`4>7OsKz$}$QPTjnCEN43`RI)U0FeY+gHxc
zj879rRGw0d3%@075tYhq1yrK(h#)h=<5A2Ko!<ets@i2cn=Do?6zC7ptR_Y89nE-}
zqHM;o&VPyLD&zi3A$Aq*3P|Vg+?Wy4n3_;jvC3ulako%QsJpv0Hva)KYi{c38k^Vs
z5sY%Q;l$}&g`G{ys8OrN4Wj)yr80>`Ciljz*;jg9<wJ#CvU0j1rYi}<egbsKuw9Z7
zcKlJ0Q<G6gUl!vxTbs)TS!ldD3v-V)Q=z$V&dpT!bDmo(hE_OfWvXlWB?nQX+MqPX
z2s8`Dbb*$8w*PBpz_l>0NHqZ-2BHz&Ex?jxch4{05MrTp;UX-q{S_L4aep{IRoGdy
zO2GpwyUq58)aba`h4ruq;<h8bE`=?8+^9)VI1>5|hSxnb=e2mhiIE2DWfDDWcD~I9
zD7q}xX$a>ofzCWFr0Ck?<^VFUxFjsbKBAo`w$$jK7z@9eGskKL&IwRz#X@s|S@3cn
zI3iAJ0(#D?9<$F)47aajSXD`WCgz?ePm2Oc`8yc%G_l$sI&W!lwiagoQE=9H%}fnI
zpflw{K2yUJ0y1f`CDb|GO7rF!aJqtxJl{Pl%u{j0o*5ajRuoQm@9RX@^^J&364DGT
zJs)&q>B<~4-OBCD1!r?q?z}HX&JXIATE{YUdXrS=dsn0<!;vFn*bpc)-QGW$|2cIK
zWT`Vzu}p^PX~Jj;c;t(#6^6HL2^w7sK*RWCY3yc+u&5f3!E!r(K|p81u8v3W`r>Ip
z&V-YESu0~sFH8w(U-m*Xf{uUE9r$4j5h1HZSWSvkJO&CCZ#1e?6n0Jn<ziB#&$e(5
zVI9O<seN&PAWA?Am^>8XsG-oV65Nu=jgeh0Ex#zZt9URS_#7#~$@g>drJvo_<$|)w
zq@a_Xcu|POpH^bcZ!#mp*myBwz%g&g#DNOCrdfIP#PgyVrN)koRfiFDZl$aq{v1H&
zNam6Qv8Mpbo4ab9t?G1qsU-m2T2du-GJtT-u#;fdJkLaYF3PE60Gw1nt1p>D=`I0D
z{UFH2Lm!nKsw@TK9FtzX4LD8}m<^7cj@;KIyRrTFT%@^X>&754*Khzhu?xzfqRg)C
zh3OBdVRjMh+Lnzg7T7w+oO@|dvyH7lk8HqK66EBuOc*^PUJ_t#`g(h!V;M*vZOQD>
zHLP1`8!gKpxtx6HMPt83u(QyhAZPLiF9+l-P>f-WeqIyMMc!ImC%|c8szt}aoS!-R
zy5)i0&N+KxvqSn5eg|guYW-8=6OpurI{W|gRv_FMX|#yOTUp#9&~=~--K4tliXi3?
zB~f}(@-ab<Bqasew5|j~Hna<M;mXR8<|Jv1#K}X#{1Nkk)VRM0$;6AZH@%&{z$!p=
zOO~<KxM@`r$+jdBZo8Nzm!iIUo*cV*ZTEf_;^b;oE(wvsscYnrRyJZ|yETDa(^Kii
z>7*BJtWZl-?jIVfo|V=DrdVD@^Xz4w9I6-0?FQpcK?szir+8IiTAJ0FZ><ew88Eo(
zR_|cEFU(ad|7<801n@Z?PUDz$2wOlriY`m(ONEf2UcSKQSd{BaFNY2C=u9Pk6)1tJ
zXRNa>gp)@(OWlnUKmwyjjAOgs-u#>Sn<?W$0pvcJv0$Z<ge_m}9Z=bb=D+bQZoW`B
zbMH4DW6^4@;p}3aQT#*pHJ#_}CBmYWzr%J9QF{wDQ#we+a#n!Li2*TpW1^)c|FE}%
za>;TJTN9xOzZEajtcXt&NjjLbvqK^~ysR}i=AvXh{vevstec086?C4oPTt%(dJKT`
z?5&oy(K}-tNY+|764Em3YC(?FS{*1>2cu&=2rTHmKL)5*MOsGM{KPxIrTcp04-@#`
z0$^>W`|IkKTMLDm<Pb;mR#kwr?qTVn4)dTOXDw9#Bm2S=L69u>jqvO#!sTJq88aeo
z7XT;hmg<@(?-T8u(TK_XRkn7!Oajfu2{*x1^aMM99i{21xQ`XnCBxNyTZ~-~+Wbt{
zHV<1nA&rkJIDak09qR)j9n?j-F#C>3H+YUxX<zI?{@27<u@#x`Dlz{C#G{_3m1DAk
zE>|B{DEi_o0p^3*yF2FF5ZbvW2O?D5u7aFvOUvkZK_u-|u2<&%Jz-?l-^W9hS7ihm
zkM?*{gn7dNh&>k;6$|_UtkEXY^u6syAkK|xDk(dj-57}EwdL7<6T;?}^=tpMI7Wb3
z>8wwU+eD`RuhsYPO`+Ufc|RG|iB_kd3v~HfCbENEVl!y-kZc^dBPqn3`k6jzcX?qj
z(ThK&y28$LvaZ866TnAC6;0-IS!B!{V)j208G-2BEY=Z`4M@o)ELAvsb4S3&u@WOi
zWZj&N38%W8BUQC$eL?@YEOrPim-gRo2?$pc7$JU8I4dkqarnC6@T~yJor9#lcvOUA
za7387!(L@;KueURGFBHC2%vPyGPo2oMY>t%Mkz>k8&J}fVN(=DIEgy9q8Ky<xDshm
zGNO$a1zF1N@^{TEs(;E68+gSA+k#}R7qhD5Xm;-Hz%1(urw)sWq`>_w{=VFvu!d$F
z0pJkg&roobOpa%YP?O<G`{xSh^*uJ$-vQdW=$+AIvVEU0OC~LH*m{&=+>U@QmQ>*7
zxK)5tf|o-4D4JY2IgQzN$|K{;hV$DH06%Ec%CW2nSe>5EAZ`@Z;0U9PU3Z4Ii1AQ3
zol`elt;gO%oxNINEd~TwgdK(HIw5^ch~>=5CpJdt!UU0Ro0KzOW`tf^izj#CpR83U
zLpF+d!>(X(($6MGR}XF!Z8`LCbKBPxZrTmdjBpE&J<}Sy=PAi&G{a+GyeG_MO7FmZ
zA=<2YAm`9~4<Pf;Af&GNg%FsXE-yth_k?Jy&X}zG(i3a3>R!NJtg|4L-h+99fhOT*
zmCFoBOe1yzz4YVUx8_R@nz+*+W`|ael(}`Jge55*Z~Q32|1(J9O%cniy>kc!Xljgm
z39=-xJK@$%rvMj;gBK}RF5f3dv|^#|pV>D@Si)n19bX8cHE_B`y==H2R5n8-r;&08
z*caCcrg6258y`<746BS`^;mR&2s6P%gz?Ht0{FL3#0=nd5w2y<$Jqujg8$gWbVCKr
zK`AyX5YL<sFauG89j#5o4!P<=)Qiv^oS22X%N_BR!WLGK>^o_jCzXJ*X_v>wSw84w
z^+ur-lzkFw=%+>bGxSc$ir^bT=C+Od!#Zz`p=B`l_;yV99u;Xu(T(zQK|CYGAC)nh
z<}kPv)7x?kT{ePcusytqaXr&D_!~@aJ#c;vmx>3MT%u)?jSeWJ$>Oy=R>}z74v$G9
zoG&g-b>>)$orGl#Ku!;7HbN{3T3X{$d5=(+9rv9K34JlG`Kf)m$EiY8<>SZ43(bE^
zg|t$fuhaj^?)+AToe8WoTx)(L)ROD&FUDq*A>Bi0iv8{F>E?w%_fs7272~jGBw6D7
z+2y&rh~!dXZ-{T0*iJ~23N>_n^KUtYX9jpih;!x+SZvx2>73I&qGU%fTMuBP)o_f%
z{5+>1OEh_Bq$$M~mHb<u*-1-tfIzqTR@R<{I9GrZ!&sU_;!<2B2o^cani&1n=KmeC
z6XD@lK`uF4zwWq5v<H^osN?Y>BHT=|n{Jci$@r-GEi3cRc(yl>!@{DH28YF`fSD!C
z>2>u#)^0*w+(hc(aQW>fgqd@Cu<1QPmUdS^woCW`?#q)$jVRmmj#yWi+1HD4(IeU-
z(Vnmpr#2(-3|mLMCfdoS1p}^4Yk9J0(mIoi<5lZmw7GT^s{8j7b{4W7jG3aH0zE|o
z_M<{98<~Kycj%5~raFZRdaQ8@TL#@#c31SSKM86~ZgRm;m+@D-P$$H3xt>yJkM8Ej
z3{%W1T%Ho-hLnyllgZm619^)2u3xT>TjNln&RS=;s_`cUnSZnZa(p&y8VEvT$smu4
z+>aHIb(u2i6_Ng;(1t^WO57yG$rieGoKTHrrUS8fF80Lf3O3plo@9RzjWmmc<rsMY
zVW*|7Hdjqw7LrwA71zcO9+cB;%j)r_XnoDm@x+4(J6}2R(?du<38H~(Q5bt4;<zp2
zCdAM~3A-tj(Bra=6_8Z}L!X>k-6V*DvXj$^)x9Fk3YUVr<5AIMjdmw~RoHcteA9L^
z8$I)vJU;VdUwk9l6~j!;PDAU!B@P3$Fb8;TceqHi)rL7QehLLxBIt_MIEEb#l4XS_
zHSGc8Az^MXs&g8uH1yjZ0cwVOLE})-4BBXyi~Vu$kx**I9p%{eD8g{T#d%G1HIa>F
zg>2DL_W_?SxKT*Xs&wKtQy8gKnAvYu*t{|V$O>o0qd`1gpgr$HJz}Noc;U($tHVnb
z_E)Hqr4nB67!V6euF^EsPLF#9@<}V^bt@iAIIBGN+~qh*fJ=&9#X!t)Tu#$y*_Dbm
z=W;KWv}B7M4`#`?F#2sNG82}jv5T62<G?6x7D@IzAQGP|Y-!^Tx>e#~U;kqh?XJ?n
zG>S7do<LlhRr0`wk0w&S{6rwyWIDsXN`$i=Bin}6PJ$+%sceJRQ#fNurrz6T|8)&k
zV`qge0v$y6#^WagSZK`oG9jn=pL~i_YZ*Twu2tA_W|P~_%f$<wmVY9fGHR6OHS`QX
zdTLKwtgK+8W>S6bEySG!PkmT4l;Rsf7A?AZyoI4f{XrNFO`FF;Y<wn^i-CH-7Jn7(
zMuAmIy7HJ7(&Nqs&$@3%wa$w^@s1#uvc^o%6%+mn;M{b}Fx{L!Q&=V@b*Vd>G+qx|
z<6+Urn~PnQ`13gs$hRBq!A%M~b-b{#|F}nhQ{oo!*tq)KobE#>ADf?-f1rgAt^a%g
zl4jxG7sD>dW0$387%AG#M6(elgxIgej?Gvs$)`tAbXLT)wjJ^}0E>Z|O-koQ1d$D%
zrwcJ%gg@Zufj!*`0$fcT70Q9)MHhm&96fA<V*Eupjge&`wh&Ev7)+I-da()Ae^aAR
z1U1JptlAdWigH${YSG}Xc?p2U&&$eQDQ_)|PulS`-yXf9vO?=2OPv((+^|q5gx4a}
zbOV<HWKD}ZW^84?7Lp}hW}&F5Y`x3CNSd=fhQgghIR=99z2>s8tSb)SZX;R5#i>G_
zozBF{@#o6{D6mw+@nS}>=$R0Y&fp(2La*0q?+GEqC6CUCD<GV{^m$x2SCkE|bQJ7v
zD)Foc50-i$4W(yt`0|Qi4=<=4<A|#uv&LQUO72)ej?shL{PRVc6ZXG)so{nuLUYrp
z#lNp64y||Ev}y5^!e)&JQaOZ6mJzF6lcS8=oSh~?t|8Q-;~=iuCwE!@7Dig~?6~2z
zgd6?Z8l)EmS%%!~=Pu)$0*Eggl~LCbb{A&Zp*66_^?u>ziH#KULIc_t3H8u~?{^xx
zTt=O&s3g}ia&*i&ldvVh^W!l>q?%^|73l9pxGkgFESWZ|J-S3so9j>%B{i`n4#>j1
zoE@hs?0m7&<4&vGf?VDNH4C?PuqTOsHbIOmQGX(%cv+w;g!Pv+t?vslFD&*@a9y9%
zs0;AYCmP0kJ7{KSi^!Ue_Cj_TyUh)t@QPhbT&=J(KzYJr@EC}KxSp_@EOe@URd38=
zp<FK_x<`blyjY<9S7C<h_TE_IAHRv1bRCBaadO(Yq!r_bH~E8p3}+QoufUgE%y%<{
z>)YLJ4FTkWPs7+<VPayHUW+YnF+sDK6jMaFCDeGTg~`<|t}dJ^P3q)wTqu&F_M*qa
zLKl(WBqST7wDdmfR%o&-RQls4g<Z$2bkoJvGj4MNvLmlE-__z0Az7=+#XIlqfKHXI
zCKvNWTg2GOvNwrKnlXB{O}bvR+n%13d+iQL_hxp9v{ptUzS{LpU`q#oNlY^L34q-q
z?e)yN2xonJptDW8_zwiZhp)MzJiZ&6T;wj3v+9u|NiNwq%Pqi>0@QMJ)8QHsq}_ok
z|FC-;#%v~@SD4|@)86QNq0AEc2?pma1i0*6Oe)2mqO<m!o_dIOA#M+Y{p}owFyh~z
zB|~XhB(r5-@O~m$Aj~E)Tx3@AvQ~L339!DXVGWEmH?vN!{(x<oK^(3y<zbgsh~Gb8
zf&q7Y+$+MuLPeX*1RfW}hb=g`I{lx7ojWEzahXWR_S4$cSnMGnlBUO@782n`kUUw-
zUOq-X4CsREIFaS~W<p#D?#Z#wU*M7aJ9*mSG=cT<Sb@%aI{klpkxoX}D_Ah>C&b+F
z2vmnAr4zDm810yTqVOm&+<ejoxd-hNRZ)8kSk;RON$m7E1l2gTs$-?EMLCzA!L%zY
zJ_+J3MR%$DonLlq(}bJ7b{4^}i!?LI1v_f9{hk6MWp0zuL{AgtlsR+PK>0Tz76WRq
zmJxB4fNUv-PNc*;w8>=X6XC97&Lr_}t}81lNcw|<P?h82E5%~$=`F^*PZNYUoux~S
zB+_GFelA&y`-QrJwzId4x3Z7u)7+ztRi5E*7pRr>Gs><w<9Q$#f$gWPu2z1*X_R}J
zN+|NqqG1SS4QhB1E}w2I)2&|0zn8BW>}{8P83ab<X@xqzP^5c=IshwYJQ}=`r&3GK
z2zL`e9^8(v$5|pMbjOsqKtY$1IZ!u_B;?<{3TmFq({S|P8offYMvRtEZm$j%M3O8r
zFt%&{&vcZo!rs(`Sk`pm{wD4aL@$|4r@Slsyn^%oc<(j-pw{&ecPTb~9m+BoDAP#e
zF#+x(9q57WmT~7d^H{LJD)1oz=8<FS<W`~QEg&+K?&tx9v%2B!b8Qlbyba`XqAZfR
z{3=4S*}tAPnNs&1CDi$|s!YCQ=r#h~mT6vbu}CM62?dRpqwRNc2upOP?m`?asL^~|
z@KyMj0N0+>PRUmM?00i4xlh!I{r$oup~>2@JeyHm>fp#=(!cnRn$V^>Hp>Xj*0GNW
z4NK!E#<>c+u;`!IW5;<y%uI6iA5-1~Ff#+#Kctn(I`4y#cmX?@I7I}_--_H)%<}=X
z3oXA;TJUZm$cbVJfa3Zz0a>4C==e^gdk}rT8W+xjwrpiQZ~viB3v?nZD^NtdCcs}|
zE{c<o524K|=eQ^p@ceY%zrn<*jDxTLKM3c_W7RSAqx^erRN#R_w7;j_Yq{7Ly9moy
zJJuB8t4(OeAGNA|H>rONh+MnyQ}n_ogw5KZE=#>H+A=Bk^>a_A6zhEo<S`0gLCFb8
zYM$E&ciPyvq-$-p_)#D=ZI^AW|ADe3SfetH|4V?g<It@yj`=K)z$p_Z+>3t>;LLb`
z7QaYov095ggr|yRMYi7;Ce%=)PXyz^imS)>3R-OaJUqO^mylW6Scu1eL}y~tVL2!0
z@yJ(T&X4=AsJQXdjF{pR;pRNOWEWoxAZ=OTBa~gfhHSJ)&QJzpKOz2tYp)pYPZf}L
zS<Z?o>}C!S<lNG1*?0(hW-~SN*Lvq!A(X1E7|VW3&}`c`1=(LO3~`{B=f&*rAk2##
zJTj5K{JsC?W>7i45uss(>R#gq!kGv%E5Z0unEBL;JzRo_%8x)!k-<otm465!5gqcs
ztZ+V{q-RI5;QPrDT3PoWp)j)Qst(BA`KmwruXgN&_EXqZ#NCcKO{DYYmR))>^`KvX
z%p&`9ZDFz15=N_Hg@j;V{2x@t9nyl|p}+p-8+0#Patv=`-iR?ZtR>7}4kV}8Q5NEE
zBMq#Y8}V*wew*3fVHG>uMyW<B#rRA?*x^NF=GsMOhmvS0>S+p7FHB=N5T66WxwW$*
zpr)Ri(;qV7a+78BTu^Q?U3l%S$IAjpY0Q{Waq8R-kDfRx9#EM4x;ThK8!yT-<LRWt
z@9=qm+|p5bcg3GY`$WFnsvR?$57lgUOq!Q}$a6q@0Nu%(Z9V|<Y#BQtj?cn)QHmJ~
zoB64oU7D(Y65@QBrYR8XvH6`E<-x-M%fStY0A#CoT^Z-foa8x6u%)bF5}S8jsTsBa
zsM|uX9_1G4LZDb`kB>w<Kc?T*Htt>!$V{<!ln<EKgm|IE_##(x*?{rvLi|CwxYrq<
zC`@OX%mLQI1gRiYkPLPrh|^0G`$%Eu)}~Wf6z_`;1u{dWev(;RwChX0FsjiafW+{K
zP>D-LI05vLdZ0F9Q6TCqU0;tqM45T~P+(9RZQ1`QJ}{J?CCc4ust$+KzULQ#u-M%>
zE#_T}aF&B?q+n?cK`y?|)O1jaE}d2FCi=s@h{>{tJ$7HLzBriqXQf4_K1Bfh<LSLG
zt`_0in99Qc_F<6OK#X%2sgaUSd+h+HOl_S!VIJ#<%G@$>L`}0NQc@;P)juvRx_8d`
z;|w9rpl2%jQjz8cGn46s_)37q&EQyxt(I_n_6r#K#dacLmX+`5xK>0qj=@;cC*Pk%
zpdE4|w`87}wij{9QqXRgGUP3kZECXga3@OX3bBm{H3t+jlNGkCnPJllE6XgMLmZ@Y
zldcx03o^?%$-st2bXI7zRk-P2V;K;avWI<HYiujPoVV$An<k;Ng;BJ$*xh(p2&akq
zh`~lQ0;i94O^v@S2bGOisEk>*=*_7UggO^_J1WG70?Y_?ud|nXmj@vay&oL2MELJ9
zzS6hU3LxZymWGjQTTw1OI*U?`wz8KXv!%zdV&CiowIr^cteAa{u<Eh%N>Jp*kdDXI
z2GL0+-X;PO_x0n>aFr~~8Ps1C&Q>ohiBN=VP5fV>E|#1Y4aQ=t`V&^MF-&1Igok=`
zs;fh1HC*qJmP3Zqv4>!%f!#e9;@=ejb9(tH=34_I8-h?0?j#cSFn5cS6}Ajy^Po2Y
z?h)j=lQN4@sTK#U36@QUwn1bdzf+(y#qlIcvj+wEOZH>kj3%X6X)Q1q!7j2AYp)G~
z{LvyZWmiS{d(Nu4jF$l7a+9Dlb{fIIS>^KnCrKUFM7VFH&a?hH#K{E8we6aAW4Nyn
zn(>96XnGtcz}$Cnb(3CziO|980-9Oro6y;o<3wQ|4${8bN<@5>30GBfG8>1EgrE?3
zx5pdQnW9`F$tEZ5hU)Qw;EV@OyK1pOOAavVp_Usa!1-4=c9BcASh^Xhos(gFsS9xG
z4E$1^FcX&JW5I3}Xw|CGIT|_}Guff&p287Lh$g~%sS>{*ljq2(C6Zx>xM>q(b&FtF
zIe#vU_Q4gZD2>g(VdLYAA1??)0PI9*`LBplW5>MwpmBsPXiTrwX;8s^F3_UT;}|1i
z)$#fFjL3(HLbiHGCqA73;R>aX>foV+MdwdKoossUCp}y;|3pB`k9k%XsY}r>%xwl&
zgL*{jcp;WlT1VH{gU&`gI-gF-&tD%XYebx1(krwM_)4fFavqYdlHa~T9+7PmR|H!%
z>pw5lovQ=a6R{zrGi;fN_TpF36pz8Q9-I8Z1X|K+>?Oh<s_p$nWTTMDQyd-U1%U{+
zj#_BkM*diO@;F9>OJC#8fTpAP!^U8Zf>fqr>MH)u6H0BR=c(dtQCY8IcEx_=<LrN?
zIviolxe5Q93u!7Cni1=4n*mY=?C)zahX}K+7h$&%=?~cA;k`Dgxc&%e;gJgW3^0Ed
z<czDmtX<2o+2$ZFLMJ-HWa)ISFxD_#%+v}t1j(B!b8ML>Fj*VjzIaJUUS51z)pZA9
z_N_pjHxnkx8~pMJa!zfG-1G-b-h^eXGTDER*cyOBq8q`hw+R2+E^CRnTmTipmEc%@
z8^SIM3yw0r6OI+)=#+#r)GGy8DmX~#^mWotnz5wj>WdZr<U~i09~B2H>_i#aG~E3n
zq)|L<xg?-kJi0BY%OZmmtCF?@OC3_F?vzpVX0BNFiGuMm+uoBDwn$rf;vsGnO;t)`
z&xdb_bcU!<@GGCL1oUhVnza}P78>Q%LfisydmCRhN$w-ZkH3je3nJZx|D4C4j0x&4
zBH^fo_gIcp7@-eHfw0XE{)4<@ABD*S|Bh<JSBWGZ%c7Pj?g){!QR&0d60%*O<pT5F
zIv<OKnRV3AH5{FFG-F{0E{72H7!XF<+;`}UV?~I$@nhp!g<Y%N>}wc}cXCY4AhE>G
zj@g9>E>_r*qrFLqsom`}Ld{ZvD-5_ktjDv$+&T+5w3F%nds#wN>`*S)g}==)X2BBn
zn@xn#N_GqH3xUPaJQe<_!cGn|)8xYCv|WKH4y#MvRue%HTJdY0I@s9V02&iYW-39a
zo+X%%(j~0%yF<8sJE)2+MY`2sO+v>MO>1MRR_l%_dqBC6J*_B!b&+GnZ!qLWI~<G~
zggH@Nx!re9NILq+v2mw@E_2boc~0F6Bx|Rb494Z6T@fsk*$~VSz}NNOxJf}cm%=fA
z65*ukWlqP~Gw%)J89kZEr-LGfqveEWF)?VSvCO{@i0crYzZ^fwGv1+1v=fgG+&@*#
zPT7~Z4Jqo;t+dyR;{+oa<`A0Fb^Ae4PuTm$oeEO|)+jtE#zgmc_Gg)u$omRgg#2r#
zbD!)FMzLt2<@iNJ76HWv+BFo!1u#;_yH&i=O4zL++0s*?XO@7n0f?6$WUPaAa5)Dv
zJsu}hrwVl9*~8(rI9r&@mmj<`A}!Je+z?tisjmoeL?(Mqom!KBwHw>>G#R0g8C`hw
z4lq}0eUSNXwCKjf!Ojbj)#6HlPLJ&nHeE6+8;oZJyMUb)J<m~(gF1l_5G%p1I9^0!
z8`rp3MpHdvGPoJm0~f3Zt`p*v(xZHoYThm&b@_2)V=0AUQZ8!5^mx%unCUdzAtK(r
zggY4?NJ$F$q9B)trzqIV>H7l&TDaPvGKS-|OPHBbhh-UiNH+)^^m6a)EQS3AHY*<A
zH|qh*m}Usp8Sp7hh%1FjH#$?0zf5m}DF2z29K-oKFFVl+_O61tB6w}97Uzm~!`IP-
zjMCuga-r^X;u3q>UH~(Z+<S3r?<7GkNvoeZ`?iTtpLz%6)+xksjp9v==@QK%HM&I~
zaCY>gm7*55X-B<HEl(3MwqqZ+Tph&xDNPx7k?!w>x@NH9!-q<h3#AOOI}1maU39Es
z^Ghy=)0NT?e7sOvPEq}Eh6uBVt_LRC1|_#2vQ%2eOo-k4iQ@dTyUEg_Upy6*zK(}3
z`uYX~P?kLxeW_=5r_+QnNDcJHlhX)hz0n?V850&B%t01O7(sU8Y9N8;hsSE!S=Kt}
zSg~j`F4pYK17cxSfv!ldM$}{T>Chfxr*cTxFVdOf4!j>d(jc$&8~})T#^XWduUQxy
z8w?=xw5AoS)9_~r?b_Cbw+ipWopCUbYa3M#1{N{u5GOX7{<ww0ZZG9-)MT-zfK0O8
z*_$GC$=weHmfX7WZyeW&%6Me%Ks8C#&T4`TDQpZ?BikKDOc=szg7jZHQ{O|7%g@+c
zj<ZBNF^<3WayeV78w9#$e6t2e1CIzp2u0RFFDmSU;#~ue3RvSD4&;LBnK4q{O$bc}
zwRK0lCBoTG?#Faw;t_dt?o;UU#~Me0P!Vn1#bo<(zes119xI@|Y=2A>;(-yli>wOD
zW-593WHp;s^%n_u^Wzw#v6g&AsCks_6-jP>Y#uN7Qdx=VLMdt$we4{VH|lgC=^kzr
zL~RtSRgU)mPgGVLa>|sh`L`bjNZHYcaF^!zJO)0@87Oq{xQ9U3G{;3+Hl|I$;wJ!`
zMV%tCh%O4riil0AtgB?}6h{h&$?j@7UdqBP<0eM&MB^<O$<0vM%yY(?M%+6DIRl=h
zmH+2;Ps$_7u1bw*f*{uxqYSRpno+p2K;Pw|GeJ&g^r)7xTs$iX$%e%KVlu*`ztx1g
ziH}VlqMm7f$nI6f5Ti~eidq($pUtvxE2SE83b?r_^`iwnP_#<}qs)CgD}O3bLtKo*
zJ+ZiutOr%3=V6qjMIY6Sh<dn63p`l}xwed(6xW?bnCeP;fh$jkaJJ}CyJN-~(C!S_
z{Hb5@=K7g=96Y1+#8lC0>-5kVa+V1un5nUl2y@u!>G4b<WXxG({QEcm>I|+F3!m+e
z(Nq><7ZDa-y-MD3xd8LV&Yat9e}#7Hyb{Be{&-S|Kk3!^UmST30Ia2H^eGYk7h_vY
zdp`Uvpym2cO`y<SrA8JwH_xKR=)gB4&I55haF~k1v^z!$awpI`TDTfw^?IK`Ge<X&
z!{6fhAlYCmgM5+ZrdH7Njwf6IU{Q0xhxIV+{~}?`2)sFanSxGmGPl+-8Td{}R-WY3
zWyIg|C|DtA<^P-z#La8M@uI?6XHIkWVC;V(kU3AzcG2c_#koz8t*M+{GFIFs(7Ds%
zN4JcLmW%S_^n#?i#f*!AkT%2O^mtAL#g$VfRtKMmbm>aG^(I5`IQ|kaS18J>a`avb
z?M!&&QBC|zA^xh)D=TSS|F(&-LPApPp0~X$kB1vPY353ChA__?%$=9cfO1|M>|&!U
zppgtNOJl0S@I6_c=57<=h*R)@63>Y?3+drzc!)2Wm{eBje81~T;<<WRvc$zAoy_P_
zydiRnXiJ~Q$P%{}k2k;Qz_)VuRm5=Vdw3#=5=?0#?U-+Y94&-06|mN*#NEyR)BPus
zY$CcRf&tfDSM!CdSeG}M2iXm6E6`=qlCeV%=MK9D%nWmwj%H4@hKLDcW0h+Odl)Wb
z?vzZ=v_%_ooxkB~cDgpxBHAqyPe<^(hXD93Nr@7}{to3O99j`=xYOpS7PHUf5Au>`
zq(foLQT_#%ydKhQ75f={7P<l2<w3p3@TRGLGl3cJxHIEfX=w9-c_fOilLc6|$!UEp
z?h#^M<QGO0rws&ezL7sH8@VCD9YZaa_y-vL>B;Kp3j2R{2!3+s5n&$v+nCMpBCS4i
z+)bda@_LmwT1n__K``2Z_CoSm`DSPj*gn=b4=uL<yO8KuIepc_bF@HbjI9_quK9k!
zTftmVUWEWys0p-kL|cuO1-RsCfioD}2x+v$LN~Tuah)JK083uZn?z)1XVq3N>1nU3
z<~H;;{xDa%bEBz+qaJ~XrdIBXi?VRb*zxhY!sZMe?)VWg<aPkFUqVGw(j|?AkuQ@v
z=J<CI!JM+`l|#Rh5Lc`2(nhyvi&>`>8Cxsf3Cb8btr{H)x@b&vJbgMCbwTjQyIFCw
z!Y%+Fe)S+4ONoW<0?e8QY@nwZEyUT%&r_Tt+MH`Yl1^Ov?gld}B!%L=E(5;@g}Tu4
zir*ctH39U$w24#hAx73F<lP9az7XW(xq~IC@)*p$c|x>TW~Td_KygsU4IA7?3^R!|
zr$#|E<-}cW9H1~oVr=T?z9%E=Uxb*k9`t&Zc&iDdWm(LBzrSxqf77n8)24xU$6)p!
zZ!6>Zlm9dVB~|`?4?tOX104OZbRGI2h)csoYdv0sTLNL0JEKjusT%la{wc<#y4rfK
z5X*+kmUwdGakPhl%(oOx7=#OOQe35COQEYFQw72k25eAwipr9~)rlOfz9lHDQ9bp!
z&?C?uW3We#KZ+z5ZUbt|-yQ7&oJ3L90Hk_c6Vg$LTJMX3TzKh!sJ0=Gf=~%9V<yJs
zdAKEBRM^GnW!p={$MOuAtZ4o?YHdP2yb3Feh93n{E_s#2cf;dQ4avfSPTp1*M!~rE
z%yrLtY%LlFdYFOEQaE)kroaD)u-x>dAPtYh-{DW>Ag4Xp+pC3En_!WpqV(paI7J}&
zB<ma^t@I?c3q<e62>&nvmIdpBF)d@`Xd#lt_(^f)Q-m!~Ic`zET<>WhcOdCW<kFOu
za<x!b48CsT@6AX!OD=b(C3?*B43S)$9Xuu*3yN+Crotg9+uzc2U=%)WVJ{SsP0cLo
zo)qc$wemDymS6XIFc+v))>d%~0gNg*zu)5pg3gc0zYq_KCT-nbjU`@$ppMXOaszky
zm!K$gE0g0j3R@)II3i*vyT{9UTuzF+MVYIXG2>$CS0P<PR!(R|wTwWg(d*sG+^>On
zbWT#s=g^Jx2g&JT4ztCRB4LtdQjfV`hj58;bxREPc00TQ<~pfyNxs_MWyrbFzSJ15
zRU{X^9dm=YuK7_i##eWHlPDBf9v#+ti?F3AGmLutLx5AnR78)lyqF<vtO)EuZ@IUL
z<AQ6uY8#ks1)5`;rQG1s*x$)huN0=EFqr*a0A%lGv8^b}vlm{r`WGYuV6{??+eEmQ
zRvsJAi6SN|de*J+g-9e}r?yEa)c+n3g<?e*$0+P{b)-L2q^E41cj?NaPX8Vi>{8$w
z3B|9B7Zd1Guq@TBlfn2QOS%j5(i7izHht`dE>+lZd$<IhW|`ZBS$qt3@;f@)2OzEx
zE)x&N??h8d?HA%s5sofhK1F*<h$~rBSB>fB89}NqEN2#-Mc8GL^+%~wcihXdjbK}^
z$UIT%YN|y`fHU=vnbd9Z0X^5g>W5%vsiH?g{w`Xzq8wwce?vJ7&U&&rn`PYpIZ{sn
z)7RKk6s4G|?)ATqAS_9CiDa#gu+GPhAqSSxCW0K;6q3V%qFhF9Wy%9=9F;|2Y^69i
zi89kD_<acfnlSYinL}Xs@d+eCsn<GV?N15A44V5w93jGm>#CsTy01ysbTQ_$EHdi2
zO3eNrB3u4kt&;qQ2;p-Mn0O7bjz`fv2J=eyAkRI24r1<;Apmv!wjlG%<!^4%vM4*}
z3otjrF6=^{Xp${0EwS#G{A-cPS0(x{bglaeq#Uw9j)z6K1GEpsTg~`V8?k+3(tG|Z
zqPX+cI&>G|?5_>LJg-vF)#v-B86$0mR}tXKC6{#mhbA+IeG8oRdK6mh{?=|H=&q)-
zZah>-<5PJEp@G5l_kvwcy<fh_caY{uPb=2qH~}sxUdm{$(r?@^&;n!mO?zQk_q;Gl
zpSID9d=HT|5yYvE^s^w(KN?5LUu*15egHLBl)3|-BdOUZ*lfsmRy8(NJeQE;CG~<W
zgwTT6GsF%Gr@WcO?-JnwyKLPgDgrcK@h|=1qUyTwN<Tto;^$OsFg6z8JlLD)jg)Ck
zghwglcW3-h#G!qmhe^jMJ)$ihJ)hClDXZxR1v=%f3WhQ9h=7KW>#@kskQN2w4g-Nq
zB4ZVymYE)!uD0_ks33A{c;ebav^m7$LHdT-e*w|pLbm%UY*{iTQ7`ffTZ<$9=P0cC
z;$nr($aKiBMP?#E6~|q?Ujdu}WzkNrI@`wXs4PWqSC}@f+m$SqL=l%qePk<4GhA&B
zBgqOA4Gg!Xguq@I<A|;~2~$b<|LlzOMVOJ%qgIW3GqPMkyMVifx$=b4La#3_65{w=
zhmWP_hBV{4#9faS<^izqu;ypN<qn=8k0h9RjE&Po^G%78rki9#=FJo7OGD=~^8sYy
z<%xNE$Fo9QQ*!mp_$rxvFVvNe)d|`NfpgBEN6j8w5!yQh)O_{I7@I1qECA$OP|mWf
zj4PT5M-y7tKDr=rT)uARCQLdO0zmYfcyhyhswk33FZj}Sy5mG4mQS6<-svKdJCB#d
z2MU`7?3lUS+Y|FI?C*6xtORQ@RERUAcJVxTrhta@T1LfN%}<JbESKvs|IqvswoC|X
zh5(NW^kuG>(n!7+Mx|>BA7d7QFeig@je>mk5M*J{2l132$2E~6vlIPtRuf^VJFaDX
zY_e#cljcjk;1PQZBPTB9wc}P4!d;<_<&4G@<u?0bz*#A|FkOi`1bD1S$E^(wN$j9M
zbTTv$mBk5~YxEE4pzS<C7S3d)d!J~hOZ8#jHs>&aOcXjF)r`7q6Ju=gQtWeK*@s=S
z#G0{0{$Vo9(VgWhmdqpJH-MGP`vRQ3?p(xwL{r;ER02IqIb6oo+A_ll(`bt6J=MoV
zHCnCpC}_Z!26W*DGB1J$3L>>uEW=+_*ww?FBsY2=3CTFd)}RmzE|W(o6tVCr#y3p_
z?*pov1(zj`vu5CA$)l!G5@;D=!;HBV3#w_q!Pqy}tIR-03o{StV7@2*C!|p$ZB!&?
zdCN5;$x3Q4O3UZT_vnEhb*oE-`9s~k!Ov36_B$Z6rVDN~TUI9tbRl64PbYILR(34H
z-7g+^@z!^2^J_bTQfv4@AT5!VQz4dK0m2;<v&L9KWL6m7X=mO%NQje`K^~9y`F`-x
zD}radY8}Ta>o`G{7y}%N5sGx8Ftdu~4Gyh_tOU@QXE}3|2Z&M>ggP1v2adRxSsB25
zQR?_vq&tl6kRvZ{W3|TGs{mU%_I;xD;xz*Osa)ROCz4hsm!5_AUX(NH9@M&E?NxJt
zXD+DE0|mGgG;y}44EQIk2IeF&dy0ESrcncJ&>vTaaDnuY9|t0u_-<MQ)O~<Phl7zd
zp|g4{bnE^Ik=7KL@7=PyLF$Wl>JM|-g9jIu$7|)uaUfWYLq$6|*~u1o1gaEQ3UrlN
zAxJ`kEH_Od`0Yu{g6{;Y32^2WE3FM-R?(=GV=d86oBbEI)f6DE6zGQ0Q$<Hok2{1&
z2r?i2sF?<nSxbD^j9%#HXbo?WBOILtY$1mJKUL=)U|CUgaX^tENGvjfq$LR(*(HNW
z&Pb9VyqWiA=1q7r%$wQSSwP7dBqw3X8Id3Y5+zHPAUPvJB#QzPzTfH7@813X>pHh@
zhw4;aU0p3g%SF;ruKrUfM)bIeQ=+D@YmNewC3^lf3IHW1{Qj+g%t<6w*_~mmB9OwQ
zSBNiP6T)ScdngiFer%D*ROE1MBuaoyA1B{`5<mj-?i9C*a2H_GM{UY-@{%xfk}Y$+
zUHWV?{zfo0NxJknavYR%CR^FIo+E^GQF!w_j0jgwTCZjdjxPZ2523GlNPs(<-uQe=
zBu!&qccU@;1k)^x9BOb&NVHqLt!9<`S%RDgzS-%|^G*aXza*#WehbQdfv#bbdz<MF
zV5czm1FmlJIId`lj&`v#PM%bzqCL}y{I$+Kxd3s;I^2hKFHa7YAi@GaYm>Xj6k@ud
zD6%+5;qtg37oiL0q?Ka(wZZb_z&1>0y9f_&uT#WCoUcvSBsTAQV6G-dLyVB^1+XGk
zJL4P$Dfo0e6JT6hL_}LpPmc8goNovB@#L;F_7g@A?8O)C7v*;}=BT{ruh(u+BqHZ5
zqp{Y8MI!3MD6qIzE{tqDSwG@!5qbQ#ca6jcqG?#2)U!ibdn3r)6ZBYb+$6vaUqkcq
z7m?z9g}HF{jUilOZN6bYmsN#10a<DCz@32H^`z^@K2->n8e!jbz|RPq|Gbi!+;80^
z$R(^`l@e{66bW?J<xeiI65>2i>P^Bct^hi_tyBIk%0<?}{(?V;bb73nSh$I%3LM4O
z;!Y9HpGV@8l=_Y!|6XCgblIjQiI<>RHuK-?!}{WgOt6AXh~qNRkg4tuHYb{E&Fw37
zjNODdCk*=JFFWoLL`itrtR9b+e=xgsp!Aq^3qI!#p<S#}*<i~e739q%!fq_cS+RNH
zd@{O(z+hZr#5W4N_RN^Qar;)#9(9bAsa}q|w*jOe=_q=!Z6TbkjtrByxvDTC;Geo0
zn~QKwhA~+i7#NL4`Hv<lHqO;JH~+)sN?uqI;X+ob-Ljhgs}RP;aCf|}AkBxHO~kep
zw}VQv?$}r(hq(Y43vXW}F#^M2MnYT=z0@dsv%M6F>W&}lYcZ#aCb3SG>09kU*j1Mw
zU_Ugttw6UnkG#c6BHgt(tYG&w>y7}F4L@u&>vU0$J%k5=w?(=D^6Z9^KmMKLr)%2r
z!<~rWYV#tRr2qAWn7drYW!NxX{i?)dGLcRwe=RK64Zr*nS&k^7<XH~r0zponKBEhn
z4-0Vx7}A_L&-V)e*IFmeEExzF(*?Q|NanUpI!Tz*;)08$i$!)WVzCtY^mq+nj)m$-
zCp4D{p>OC$L%hBVL>~SqY?|?o0LMp!l?m<iT^+xzpUL)a5tcNxW|m^yQoH9^y6SL&
z2&W3a*xKA8fK+>sIdnXRd@Y2A?m+1@=9h#u<#xxm3i{6xR<_YtY7fVgCxQ<YrZxTW
zLCmse5tpq8Yh(gkcFdvb91Nee7g4AyyP0~tBqBEvE=agR9SiIYWM&tBi(?gm{3M6E
z(f%t4I;hO#HC7K4O}XnV2|w6}a9(9_)lf$oqm9=P>QbO4lu5veLR_gTLy0%gN`R?|
zd;KTuOANO!1H*Q?6DnZjgp?XvDol}(j(I4dt%%%-*QA>K;XT<7=87O`N}@n4Odu)f
zRoJH$!6RXSdHkxXNxiz-&O^8&UFIPy2&l!oH4y&oK~cS62Vs}Lljh^Rv;;;*CG{LC
zpb*A&+9|R0JfUW|&Ud6&x?PZq!G(xet`m}6q$a#2hsRACAJ645uR+ISUBsd8>AAXf
z>c#&Z%$6I7G~eXeKDnA)wA=B~#tlYAL~e7;vUIflmLP9Lc<Q?cM%i)Q{XO8GdD-{(
z#B-uuYw0rk<1GPBt5U@|8);4J1<KnA>Fo1sT}KOcQfZK%P}1lcxvo~yXshre(N|=}
zgu=4DivTw#`i0!Ft`_J<8j-GXD8@HHnC$Qy#ay~$i;`pBI7?KkYje_{UQ^j%05JDN
zBr0w~3UHbT$vStxT8L}O`$@^ib%MyI*`uA$@`FS$&(n?{<CVBUnA64B4ZYVp0%%wi
zCb3MDaI15W2QM7)%cDXyG<zAEuZVzQNH9o|jktHnbaFI7PkKOf?ly=7?~8QZ*hk6^
z7r|}fZ~?L)>$!6@%exA6{gNp>ie4ePQzG>DV3_oRAU9WMKN84XQ=!vnqnrzhaOt_~
zEn^F9>bDb0f%HTY{onwRR2Q@LcqsoZb$L`q91lSo<AkV*bWX$Uw&FC=xiBoIT*8R{
zX+X{o?GjQp9!R;MDAW-V8XCHad}IlA@Y=jom1hca(;%^X<9v=Va{#T9Jn?@Z#Dz%L
zuD%jY^Ge#P#g9j!a<^A~<L3feeGxNw%vna`Sb<K1ecIabmy+()W_hvce0Q##ztX4U
zOd+lTWz(T!%)UQRt0!^9QfGcmn6q!P0M%k*8Hs1GhA<n!=1Q0&Nu7&?DZu@MJNvQU
z0g$P05&19?>Wc7xj{^z2SZpSB%VxR|7fbI|vQ3L81nF-C@OVxH6=nOJRBK!R8roU+
zRpjNjR{+VQe2bTVL%5g@+Up3&^2YL`gMiJ!+BD8BQbOYe=AtA|>#>s%7e(p<?O4tc
zl=U%eTGL^MZnZs-f2N-^?MP1fcmBC$YCzwg_2A-X9mF!vdrbU5xN}CO+DFXugt$^N
z02+ux4*|%Nhu(YO?QoQ}0yC?0Q3Y}QAwhH<x_6^H&N&ncR`4xuSJ;_wib*-+wG!d<
zNe}33bXXAwHNU3ob|J7{K0jjh!y#NMc#lAMngFNCg(Yc6c0U4ycCTvNjIu;z<{+*)
zYiy?GJF-Xw=U!~I+e(0wW+cGjql#3y!BS(Nbhr>o!R*6Ps|fdWG?$f_d^B{<Lpo(j
z{cAz)$4FwDA2>ojrijK8K$~~Q*@E0jIZ24$iFEZCs)#Rl3vgqj_G&4}n(<bd6f;mt
zJ&q+Rg<^(aXTr8fwA)KJh_yNFjE#>gGGKN9cuW9)aWkh&+P-;@ho-$a&52HhU7AJ(
zJIa_*0=$38y#KlYGXhmKF6K}0-{U4th+}_C7>=?3Lq{*d>_|o~dXn$H5|c$hdb#Em
zVQx+)6$vaICxYZ2*3Y^?nYTR&$hp!-Fkli*PRM3l{}<u-a^TR#DZpMQ19|$?J1&_2
zOA<LiXr35lBq|m3N5Y&v4)W!S_J2Z1n_j^)q^Ce=zcUiEoLc;ywS!#16Xw60SSZlH
z?-k<0==v$guyN#RK%Niupf)=zD$diLrx*XCgp}(0Y$18lN4<zIFF7hbU#L3_CLyu%
z8IWcL>Wg}8D%y>%Z3rEMm1uK;nW=O|D$*^)X|wm!p|1`kPY~+T;z<E{qAMN{<Srm1
zxUN|8OaS!{SiOzH=Aa}JWQS^8B8(r%y>CEleq8_z)TNA=^(=p7ZlFIhe19l_<m51A
zi5#9VacqoLn7>%T`lBrq$i)-m&mv}e=!CV9b6H(DS@~?fxHvjn;_@eV%7iI*UAXE0
z9fa%0n?R`Eur(57*63ibk9#CS@(^TfH{x6YIa$`mq<LQ8959k)Dj$rx2#-<BSWxSX
zw%2jJANL4%TOdHE8&}OZ^<2Q5A<GmjzD7u~xMKmDM)O@lT?{;%uq%600Gz6#BVKAo
zk%&}t)MHm6?p3%WQdQ)RFq(p$iiTJ-D(5+s8ZY1sD9l8i)SPjVD0eI#35m-@npYg&
zr8k=9Iv>ad)iFP^qP8|G33Y`U^&Z{3xx9?0XWFC^!j}E;3y9~Q%GpL=JR=$wqN4AN
zw?$BtG-t=67eY~0j)LR23g@26)ZE^c6sBhhbz*%ya2cQHKR6wavo7LWOYxmwb7v)J
z#(kIY!OURVU;=ue1mq{P60ZnwQ=_?Mmrc=rd?^^^<kD^<ju24{FHZ8cCarf_0ix7r
zP|FUaUm!JNCGO~mH!g=VH^n-g$bTe=O7fIU%yossm7e}sA&2`oraxF=5@D@m+j)m5
zCz1@*=z71r637K+1C4w@&U*=?z-W>O;)SaqT8tW;7VnGZBU-(B%zib5J53U*(=NOj
z4Z)cUFoC|*jK_tUcQ{dH;C?57gwp+k*{^}fC6nt|*!EjN=9HXQppvGqoGg$!_YU{O
zSqkS|;BDGeob!hQ!+npnWp@k91E$iHHN_Iwf;j0mU1v{609#!L=mIwNh*6v*pw(Mp
zU^)nm^Mz7^5$@W|bv<FHTwz&YwOv{WV*t}7**C8&vNcs>Phn{(#tC-H>E@8lHr5qt
zPB$fYY$@9HvqVOM`{y`mI-vP(6-}Y#mW?H-J(jxxz(pEvbVweaD8$K5??PSGCOVCj
zVa{4RG9_7SB#sv4g6W11x)M}_vDF{_2XR)I5<7|VWTm#%Rjy=iv5Ro$tgDEWWnUq*
zFj|ggoFpQ5G&JgIih8%i(l$|+X>B(W&&6fJVSW|u|L}QT(?za0yTmZtlWx8mj|<Dq
zhtpqtua%MHQ9bz*@B9;yav9Sjc5%{8#cz^=dgF5ev~Lxoyr*v_oVy?sxw>UD9=iq5
z1(cCz{O`}uq?w)!Nu6-ZTaCakuH<P%og@TKB&To--)4dnk^Yz<g6#Tfsm*h^!GOL)
zVGjs4W~{BvXusW22RW$zv%+pDjv+KQSl#2VcN8d1qq6`Vc|0UES6Wt#jrjGQAT6HC
zcM2-8`-GW)7zNR_{w9Dj(>dGYA0nK_xJgN@S^KU6Nf&}f<3J%)WjgcDL*>`$SWRGM
z)c8vgzcwx1AAd@S>!1@G#0p)8*zIn|E3A*=PeRm<WcoIG4@7H3;X$}I$o_=`4Z&V}
zrnBN6c`qbI88>n5*y29It}HuG?Va#=Cd}1I%gbnt3Q=u&uQv`7Vdm2%(|U1e2{A>>
zr(zr}h&*+2$z*VbNT;dc%3k_wA<nrC>-nU*-{pQVcPYd<-p^ql8`lb?xXdx}^9KmG
z+Ci@u|3d)%4aI2EcXvGq?Nl{E(=(cJt}yq>e$Kh#bJ3YgS%)Y|YKOl9J9EYi(}gq~
zlO6(eh69yRR-^p|kQnyrX(H}?7}^zU<JBU%M!yiZT;c4N<3gb>Sn|!ON&HJ;PP{Fd
z;;i-v0DQ&?c{6r=)L)BBP%jAJD;80l0zOv!%eIVl`BMRI<SGi@p2s0`ereTY!+M!8
zm(_>-F~<`iBw4E<uxum3#bKmUlS>3R4UW?W(0g+9S4Ki!<c;S&|5hZ5Xr%+A^PVga
zslfQ@$)`XvOR$|y`n6iTBGk$2vPwG0d_$PqN-u}Subzf>qexf4W{gSZc!7?D5MGb}
ziFU!e`qeAec&3QJk#RLX`8zbcv3*wj<XK4bffmvU@diS2OC-hK#OI0>(&JvQi*}!3
zoU~j4#211+7UYz#*@6GA=Yi<1IK_(JDC}fWDlzmf7vL(h*6QXyPMBT*b7|nRw2Gey
z$*I;j2SdF3Mi4dNCarqTUN4sEFf$}qEf)&&Up^<S#_NKd^Z<8Q=;Z$q;u7hRlF|6+
zr2;?{!2s;ZmjPTv?ah-Dr^YK_Zok^pN;{8^YrqT&b%x!;_&KM?GzB@s<|r4zD{-n2
zXNV4_8>60A3xMesnR#gePDT$i_h1RMvLLUb%qN;6wcHNHAIpDPELb=9e2s{ici0Qi
z)MpD}4>?ke84Bic-tw~w@0NTW*d@Xah-b?iB|xKLyLjUp#PCpB0sFLl-h`yPaV;a6
zsNNvj=~8F7&k+H8@H++PR^x85n7hqe1td=zvSd5Bgz$-+Yy%$=<m|?cn;cJw)R>fs
z%bOxxSe^Xw%Jx41Tw}xw-D2HZNY-I!Gtjz-H)AFoqp(80TKMfEg(@d*v|5i43ee$q
z1+N#GmmX=@>sZCtV)z}<+)PQkG!)YWx!|~ai-X>UbXjpuiuURS0WhwWM9+KBx$GR9
z(tZi?FX)>X@AoT8P9xn8-|wGJ1973fAN!8mM7tQsZ|U%U&i4z5i-;&Su_+PcX#=*S
zW0xJw5B>$1I}7g>!pymZz-qJ)6>P;JTpPVoGC`y>vPJ`U(H|5EjH0$gi}#`svr12Q
z;F6ESn%L*VBBi!I<VKO6uCPGDdq+v920Xuv&eOzku}GJlR~*@CYLEUvj)Q+HrquZH
zN5pYeuueg|jj=-93(#8HL|ub)>c`+zv72pZocBrb8;7;}yvnBqQe}ofAlg!hOUnit
zYt7G~nGZN=K|{emboDcY5l2ckjmw)vn^y=NsFk=KDaf^uDVDr3RAYgE10r3^pKgEL
zC`t=EI;7VWZjFHnvHs_fP7q-}sn)83=ozF&e?rgyuQE=ulc$CME68&fx;#G_{;U6h
zr96=`;to;dInv)B%YH%FnP4EAyeo|eGGDn5uJLw&kesB%a2-8-F3b~vN`sp;eTnaX
zS->cZ(6(A{GP~fM4R3VFJ9w`k_wuO~u9!q~rZ{Lt*c``kf+*=!E&;CozanJ`bZnI5
z%sX}y?55&ELtFewbgM#k3v-7c7r&2_y>yW9xG-0ejk-K1%>6ZpxsL&2>?hL2V<QI>
zP7#oMusqpaDB3A#g=fZI;2RK4S`ECMz9|BhOPm}u<x(MrBd6kM^V{No%+_FP)P*=p
z6fCG#ddnz!au%+iC&X3Jo$I9Ey}OLnWgC)Lzatg}K|$6N?}&hh6PU%nQrI0*v#$1L
zBXP$pKWk~b(p!A^Xw6;x-};hg5N2Zb?o?&jSpi8L^U<#O+iWIKWAI#;2y>{}$J@h8
zeh=Cc15c_^n6zwPFVw7%)v{ib{iC2(k3+@Rj7NpIo)UTJ?W3{o_X`XyvP>eH@pEBj
z8CD_e>9!L<={n?N;x!R@$(GeBM|_ld^6bSw*r1TeHYL#H5D7;jKOlz7!wY^gLYpqc
zDd`MWyC&kDQpS^l3t{ITAuhKKR%x?M<^W3!!|-Y=5l*N%+}@57K%&e+Lj4;Zp*LBG
z^v~(vaCSH;hD6YtIDBA1d`(p54jm(j1<T9@W=_avurGEO;Ks}H3S~KPZa_0%6N>Ww
zL=df(t~U|oh#waJjnvcY<4lpa2+XN-zksI}<^iBrVT&I#6sB;Tgd@1Um#NWy_A~Ke
zrdYZ<+oC=%F<}LDnj9yIaChN=Z7|LjZLTxlNpt$S5SKPx8=@e6KLW~fhYUfO3N-m1
zBA6o7>(YuIQAX0{O|qfp`SS!j1CD968eX4&BPHgpRp;Y7rAz0ATZmvfpe>|y_<JTv
zuaA3!Kb@biW-i*dA!KrPb6W~@g@&4(@H{U%w`!%gUB^DSiY|U^<27ztlci(@f#z<9
z7MHlB1RAJ28*#k=ibkgzis=hNm@Ok&gY~E&XPumW{%E0s*5i_~oM;yX$AI)y8h*Zo
z!DvGH1=&Dh(wAr*y(03k>C+OB?6i&&>MHB)dYm{>u4D6msjVMvB_dm#OM~c?2(2G7
z&0sty+J)BJzcN2~cM%{Ot*tKaT8l3RMRCSWSUaXEOmTSLwGpRgLWWrJx`@mwuA4Pt
zfyDt_EG))&8cltAIf3d9I;4!JM7i?$$qkm~CBV!u96<EM|3tfZjCVQ8iqR#3%m$jE
zs@)kg!OKGZ<EXf}{Zi1*f3VhaT6&d0M`G{L6|XD}?X{)Ku6EuZ7ysxQB{M0V{m>E|
zZeT#i!qFLr33Rd8<>3@%D2@~4v^vn5JS@_k8l!TIaAVzNiht-W+}az6i9acz?%_5j
zJ<hiU<sOWKAH5A%HE8bgnj^K{La}Iw&N^Gn%QC^+uv|JWlSFx+^bXrEM`V|b=Skxf
zk^GB@UyoBom?NBr_QYb#L#wqXj*E2_Zn4l-NX@9N0O&?TeO+tF@#^>$!AOyH4>h0&
zr^q>fe_SY9DXu;--pJn&3b`QG8|$uA#KnozKx`zMxcE_uLlriQ=`A?Ks>YuLX+mS$
zJ3Vd}We&3}>$vzOA$efyQ5R`Ez7(cCS7#hNh9Ifq?JaIqIQQ4&&-l?2;3^BJ59hB;
z4CjxHU7E}95#*lN$!08H=$N<)DCMwYY4wgRg_s50+QR-#G#O808h%P)8UxL^USIi7
z85P5cwPNd4{h4{W%6qWqi*zxO3z*BU6hP8!&!nA?r$obg&gm=ho(OY<jknBszqcBQ
zo3PQ}NXNrNae_ccL$=iW)gKCRU8kZ<)d}r4WhCC!(H?Tb>O^wwX}{Knvr{Pb@r9(=
z^rw(&?THiOTZNs^0OrmZVl2HTNN!W+DyD&$B#0lhKJ1~e`G!`W2V+W_<J{5sp!~gU
z7*kg2zx-H!X%&h?nW5y+eVS0Xf{A7`CX9n{_vf5DHW!&YNhLpFzn?I(nfuR-qFKiS
z<QCNCWC_u^yI|DS9~VpjAP4q8&A3fOS%_X{V3w^#CxYh6)<(HGGZ>2tq5~(XJjRQn
z>PXp9SGYB-5M)%hsrh@e(jK2F?8LO$(LVC@N&a02iI3%QKNl*0zE+VG>%Kjn?I6s*
z+fp3wT7n8aM%o0WPX37S++}Loo$4$#f#wGn^T<z;)T3Dk)}2h){O!iE>xLqo6}O^T
zW4PNUNb`eM+V!VE<Sxy-d%dxP0Cy7(E_KRtoDd5WoFWdlo|L#ouzAGc3(5qf{rAhC
zc@aQt`Bn%#<}MHV)gP{1{L>sxZ(nc_D$FHhMP#1ml{rC_kZVx&ctC_J)S%JEtm|(G
zM-yTj5iTr_b2aIlC&cxUW4dNsFMt9do}goTB~uLO$?HVVdd2S&C|Q5s6GCj}6Evqc
ztPe%j$X!~*hBkmSBjh}YOEXEi^)JD#iJ2pr^u${NU8|}Tjj&?IhQw2?^uFOoB3gZ%
zqQ`GH0&rD&?FVY{jX`KttOH|nh0O%~Z^xM;9l1w-4x4eA5a)*^Q_%)?#?OH0CCDMk
z+HI~){1aY7x{w%X*{g))aU^TPct?Q75teOO^A}&a7%(3F?WP5QqaKbPZxY~QV7;~$
z((3m&1EI{kg&8j@>>{<}*afv;Y`l2^F^jipX4_m49H&K^IENEuULs&%Xru|evIU?A
zH2Pp{G-UA^Ckllnv<<Jp#+f21`t&-zV6iPB)Qrgy&HP(IdAMNdT8*7VyV&yF#<X{t
z5HmvW?+wL5Tl<$JVX#fuOLS&$@|aSM4nZ!J%q!4>tE}^G1L{$Mr$43+=GlXVxl#-h
zO#(zrVE(-y8I4m%Mqfgm51QHGUe3I3(ciFiYDaJUHHYz|7ISV3o+km>qbw%cb>PJx
zj4W0Z;G$3t?ID{nW;-w!h4m`GWxdY!1*naHl_(ntG?UP#skdw`B&RprY){*-gn39<
zW6t3%8o)k9ZB;`O>1f;|NIip<@p*+^3LGYN4Plqn7EA8{==z{<9*8@3gl=_C`VSWS
z0JJduxf-AT0>U}y=9UhLSKk@P<&yzg^00ZJFh|2+k#W$7tIKGdU2uB}+4sg0V~gT*
z*uMxPyEJc{w~GmOB3<!NCODj}#G$(uxn}c>X53g}sCO*(eJPCOWX>E%>;_@x@}_4E
zZ#K6IapAR0;zVgEKHMG5g@+^9%vWN$J%G$P4j0(dG5BMQKywbMWlHQSnvp?MsHfc`
z(mib|D^k{4d@R(7!znsQXZ&JMAUCG;9+)=tYJYj4U<Q*+Zg8t1Dt9{`Y>B%{8Uyco
z+%4KwLve^xop^PxB3a6u-qp~C<wtu1yT&Lg@WI5Z0xJn~_3WcZtKD`pfywSpyP~>4
z7o*b4^Q#Azky!BCtiJkh8HvYur^L^HRb+`_mmaa;&1_*gOH3WoRz6-4NMo_cjEh6|
zfhZbFew=<+BFuZ(?Zylt<|ZwbHrwY3$_eXnDYnu)i&XyEUKwUFjXQ+6b@Z&KlsL=-
z2kr|>0T?aSxLQOmKxIU}YwpRP@nXQ`-U^?+Ht03_!$tBdp2-JeY#W3JkuFSNaxK7-
z^=j?!%kKym+JFi$Uokwa#Gk9s&YLsezRI*%yjG?_cFUGzFSe;Gh%_T`B+Pq|JW3^q
z0wI&KsB}P>D_FM4DHQVE>Yae@c-gyOa`9XhY}POVO+|Q-ZQP?m^IQ#EaVOCQK>xH7
zcl|n)n{KV>7U6^ul68JZ&%-U2V0S$(QS&@99*U0X26JXK0(qsi{dl1+Zl#?IjcJZQ
zXC_b*=YXz}{6U}@J8s;>n7;>-jZS-KbSP++Ab6*d{hA;OiB!Z>8odx^Ii5G$Vjj`V
zl)YGJj8)hsRJx^GXAyP5w&>%76UAQvd&gSrEy(F`dt)><Ye1V(n6<#ULrS1`RI|w}
z{lp+AI1pPZ?DF?ueY)`gq}idp1bo&Earb@&3P0103bZ~~4}zNI<Kt!#7Gx0%`AxcQ
ze!Wk)6Ku!k4+y)&_)w_JJfQ7qdlQ;c$aaaY^QK6D?&5ehwjP3pt5bX9cM6gzyV^=z
zDZ=pvd3Gr77p=vHz2zecn^V3#$dh>`gh+{f?<kiRh}Ehu#U8?tq!xz{gSw+27sz>Z
zuQAddW}k4azB^wgm<jQyXtTUCZ74qxkZaxFdPX6Jr-D*;nasyCBM{Dmr?zZ<^_4L4
z(DUO%(+GP6a%faR)JX^(egx%iBhJ!y664l)UwkbB22C4{7$um?BEM9z>GT4SwsiH_
zLx4NC=3ka+=C&?@B%T}<$A~CruG@-CASVfNm6HbWEYU7L4;auDk?CgNzrY5t2IxWz
zGe$A-%ikZXC_)}q)wC!{ge^hLa4=()B(drN#N;>K2aIb)P$hc?VCX-8paF!(-hMq(
z^p*gR7sghjT5Wl<UE<e#a4%rW(G8uL_cuVUNqTu^%s~a+&Go!ENVE&dMH2di%JYCg
zGX$x#XLtZF<RDI&y2*D?^c)PNCXv4XRuTSQt@ObjHhYKoyPO-vGnq&y7cYr``*u<G
znJD)j7HABhlE2@>4h40&>?upy&Z7jnwK3&_H8TW=-%{1zQT`$u0jKkxK=MLDVYXQ5
zFen#O>SAdGw=N+@JDxc3!6is7;7jH_doIA!2F{1$5|L&P?;%Ucu;LK~f)ZC7>1~9#
z{KJjmyyu_oNI*9Rd2u@~(~LM>d5dL_D!!v-!@;R&=SvgEdLpT9j}#TdBHSfW3~~HV
z$9S~F`q=5{0kC*pn0bay1^vu=M>`d^0E6*CCi?W=)WOG;QE7_)_~EeyAm?QD*jRvT
z&Y3W)ee5NKBI24z4Kn$-A_7u|c3am7$@%xzdFtya0l5vZudP>cDRcr*9$tuF>>b+U
zY+;UuKa}LZV};)WnWr|zkji7sNku%Vt8@~&lc3CIESKUN(eRBIni_G)$;HQJjXMy>
ziDnjJXBRgr><kc1*$$(UjdxA~bm7QAhc1=)w=i;GQA}^^%zr90IgAdI!*4{m#(I04
z-MN6=eHcrdpH_UY)~8P&jV%P^s$yTREzD7ZNRX2~uFHy`4|B4}4EHBd6u6sd`f-Kx
z(yw(to77%#I&f=3Zg7V#*=Z!y(?)5tqg5+?Fuz!zdgF8bE`If}vi#-@h}K6=)rQZ6
zHapPm^$qBXTgO>o6bV0y@r1&zOdU_3axpyL*+A|!Jrxb`I7f&Jh{*+|Npi?TWxTG+
zuuhg|JBN5VGx}Rn7Of%71(G3P)Xs&5J4}x~ZE=z)%8p~d-grucGnR^x&Ns&l5I4n?
zu`9*`B3)Se28yRj?0udSkn_AaS_ILVJtS9NBvSRn$H4US9oE&?+Z3h<L;ZR*{9pMC
z2E4J&1$=Qu<?zPxerzX{1#F}v`W007o3vv5Sz(fqs8x+I7ecx4Tuon_Bil2Ds5H3z
zr(0}#5u_Q>gk3RJG*w4QRf$hTm~+WvGYe<zaIyb~Q+=33Un$a+M-#%b&zps13{+R~
zY8eA%Xxe{%R6@4D$Gh7lMON~F!(kdtekoX?>GM`amnOEn4BR7(gP)-x6eO5f2&CRM
z`UVb8E{AflFwsiZ!W)-RbAT~GNsqsxh=rj>x_q&~l|Zl(NiNn^7+xUlS7M+1p;76M
zHicb){P-`j-RZ4Axi$9MRYY(P=|%GY{?$&Dw=Cj9h25Chr>sWL3qms+c}QN+9=`{2
z%M7s^#*w0(X7XTh4WznDx^4eIQEq2BG{FMwTOm%7xe!O~opH$@3P|?nXnf*BK~%R3
z_2KWYB}{cWU8}~wM0oz;Y>hSkqU#*92QBJL3KO#jktIGC;jFPgl6lACHx&OOKH=JW
zj}l;+(WPDi>SY`U#3drkO)38^KOgW%K(i@%-scV_x{OPNng_aQ@rFqEPpM;Y`_qU;
zZv^vfmdq${h=5sgOcw0=G#eN+#q$DkvHMwMSU~;+;&Nakz`Hedy2yaQ%tmsM4e5v=
z|Jgmv1u5Q0EfG`_Tmg((ZX!-=Sh419B-&{Yqf4sGA5|P8FgH3)j3@RuQxFx@9L&hS
zxlBgJtnr>`_{Iv=6OCIST%|Vp40rYZ3}ANf7=Z20-Vl}xiyv&Jqg8GNF@usRlsrIF
zhaSQ7gkfGFJWOGie%$1>C&clh&31(QxF|<OHi*9{?DX4*na1!Na2tqg#oRts9dCmY
zqR(TDjdwvoWQTcp%z3-Rofwa-tFYU>WpG7**+;0^hc#CXeJJ(6SzyunZESp-SgDs-
zawb7mcM4O#Mlg)KGr_&{Mtmf~9PDDz?TiU`7Qocx53eGi;KbPY-Awcp-V^>xv^mqm
zQ`Fns1({2aB3o7+NSp@?WvH^wq1ZS@l#31P2h-z7SLPT@s+?t>em60kDKqX+JTDqv
zwK0`^t*|*EqXw^ybKC=Hmb1L~HQ1z27Q{a&DRD1OnEcaLF|I5lx7@6}DhLj7vOYcj
zEy69zS)?Y@!F&BbE}rO<cZy=&>A|Y*K7U|7WrqA%6j>r>#s3t}e48?E?U;N&v^lMH
zXq~u8fEhQ6{lhIHDFJGl{&+})tAon1jzi}7$pb(x6h>JxvY9M|KM>k`V)sn&z;Gq@
z%S4hscYUx(tF;L~N01u{A9|dvqz(HeLS5qN>Rp$WKUUg!kpn(FTK>oxW)kcflk@(S
zAE^mu)UwQ)Br>-JV_Mz+4}#p{$+@n)UGDOb|Kb9qgn}1Ex_-!vb@WZMJq$wqa6B{^
z^NYx1iux*n(*&8LddQ$R?i0{rk!09d<&h!^X5}19tRcWrm|!F#jtFTLExWt8SD2dy
z54`b3`4esd^ziOMj}k!*joM|i$BHCz;HyKhy@j}7a^OE2_Y0u=VW`GXzVGqk2P_ce
zwtB-S3WPyfr6(OwA}BnyC%#088Gt!40%0}2|F;6d2Mvd9JW?;nC7;M^Ty>FdXl@>G
z_>o=;I9aG0y3WJCaYO#8y^9?XZ^%AbBr9Ex^y_B|a<c7+X4i{!vc>MZ8ZQggfI|`S
zfx^xL`!DoMPeFUoO7h3V3fXV2rvYhTj+h4HYY}9TCT;FWJyRsWG1)*2izaS*f9CE?
z@ZNPLz7gRD<k--z68`Xa$L2~ySKOHidN4O^MC7?gr`XaD&-N^s3(LcMs3e{*;>d!N
zhh*_fAdDuYagan_5k+Ybn7DxZLJ?h;cUd{BvEYkf?gHGS(EKfc|CwV+?Fdo)ujiLy
z_Lm^cO<fSl{-!S#3O9LjGHz4Y++zbvmtv-mO<o3cP4bx%+el%S7xPITI)8fbF~Xey
z7Rs<+TfPlm1*8zkMR!evQ?uzg8^Du<nJ<jUM(p$&G);uIWB_+Uzse+2_5hb?dPTa*
zO0gC@z7C)oa6k7Lg+1=@OsFa0d?8+h2lYG#)0}))T_>1A!N{(d?F|TbyFs>#bBpv4
z;O$ft57k&kP@Xkpi$2YWw`aJzmBuwYyIxaKT|UkQY4$S&kQD+<HO9YLq>V{RMcS*4
z1iAXGxYJ_G{}%<5TkISKnXl}?v07ng5)TNZ-aXvxKKm_)Yu)`=;1hQ78~6;VA^&_L
zhzg>E6;Hk_V=z828mY#2C6v|yCgJ8koMt!IbIw)Rm10&vgncc4tYXyut-@3gP2o`N
z_;wMqo1=Q3BYp=!Jz7t##08>Ull1sf>LK?Dbf@T{$URzh{w0h=la2Y3??SjrQX9(y
z^ooKq*SKvQx0O-)aTcoaw&;6AfkC4P(wpb7u3a3Wuv25)CAn#Ze-@CR?Q2G#APRsF
zww}0NgxkL=uNqh{zYpU4xr^5s3;e5~QM;}cJC!s!)Z$ptPC*xNXq2ag<gr`9?mvF;
z0e}Z6`&ZulOw8#O>e3=2)X?lipCB_4vp*C$@`KVZupC+Nuo+`N1al#~hOw2qMRe}Q
zS`)ajAjE&A?Tord!kj!7OWDwM*N=*%*j%>9<DzqyWaMS1<sSp)c4A2ANNo>6G(Urp
zI$t45Oyzhv#(YvF(d-&#ldL`M8UkHJOuCR#VnZPwopNY<xky@dIz8{73Y&$Uy)xuH
z{8IpOOILB?B~h(uN*A{C)t^N^19vfT`;}~I7cC)VG1&&hUV_MXihVs?Au{L10~ABC
z>*oNu6?9WJ-6i;!z}#9quc?!rk4vbEn+BaJuKgcknd?;A>XiG1x%DL|F;Zn<`hZZU
zhg8+4_e$FS3j_m(IQTkJVRK70fqyrD0ip$9P@2wLMfrD`yQy{W5|r~ss>M97H|`ad
zdmg39PGTP|f5ufOqQGE0DaiRHHD5DU`4WT{WxDT(HNJ)*^Yq4493{$`bL^7bU%gR6
zotzMHz8BvK%EJJY12%%+0=R%=iOQX$kc2rY-Fed;`iB+GkZppSvFt3HB+hj8^+mVB
znm_8uxnGKKv9(8~TOK;AqvAC?{-AJ9iA7I)+m}j6!mOmNR|Gi?1W+Nb2_Xx;l@wEE
zGr<W`CDsw)Dll0{2pNt2gwbr6CB=xs@QE?XL|gwJl=J6_=%!pX?f-os_brrJoYJc0
z?iA>F%$*do8QnhsGV?g4#17zg0qPt&Why^%c#oRrp8}i!dtN4f9`qGNNwC3ZAv|#o
zC~73dX`Ca%D<=v`8DUV`X51y%b(4dk)U1={1oIF@eHD9(bg6XnvOkV50Ty*G5nWaS
zbV@qb*tz({++||Y^DMUsX)WBzwlqzLPYO0?5Gpb8)+F;*85aw(>{P88U(8KhDmT<W
zjW(VChfq{bTZ~vjg!982i)8H~A)YHS$-qbWJkYsSlBH-fmKQ|5WSd--Sa6~M7p7|z
zkqsq9dt9AkA+HSayf3bh%aj<TXm0xAJ3*91?<vKP=FNn5LmP;2!MFyc4d$gn%uRN5
zQ`U*8KPmu*0%u!i2xu`6?@NvNQiuzWi&JU&;)wZ5h=(DyCe1ZJ5S?Y*<VmrD!fZB2
znfMfhZM|4#^hS>;&kae7p-b-=5tpbe&t>z$jQ+Gxcqp$vGL!gJv{){?^05m*xZ}u8
zI8z~ljW9EvyZ${fU9@Y$DkJgXN+D*k&nhKS?z>=-l+J-U^&O2qVJ;^<fcpi^FZdZO
zV`2~M-m=3&#ox5OZW{4!uQ2M2Kk?>p3@;4jOsh5BG}Z81F96{dr>L>eB95*$=4C)p
zd>QT;jwKX!F|;k#JuYf-qEOn9`y9P2LdgD$0=R*6(;J7ev6P_P@^S}Mjg5pbtfuy3
zqE(HD%YXEsS)CKIMHefQ!E&vuie<)<g3MB6r-3*|bgTP}8@Eo(wL}3lWo8pw3h*C@
zCTx*KyY^TD$%qp1;xmCRFw0X1Ue)5uG9i?@$bNG!StO+Pjip69YbwM$lQRT(o@nw|
z#ZyZ`(ml{-^~Op|L%>8<(x@n`58Swl8$_5dwO($i^A^=IAWlZ!`1TcPmb1lY22ZD=
zb)l}6jwj-W%R;MHqnxQkMU)d|<1~o3j8lZTg81L6sm16236RUr{Kw{aaRIqsF!5=r
ztVO#(mu7%F7{`cIHzChs6?QK2DMNoeS%`DNmI>zxqcMK@0zv@dN?uo-D#-cNxN1cI
zW?nf%APJ9PSF!F2gkg{#o{MutxTffdQGDl*D7`S%Tak}uj?5#Ffiy6?S1M3MC6?Kw
zl253MD_?gM_k6`8ALO&4SZ@r32e5A5q2lDFY+)`beMc;zqaG!Yzp#UVA0k|1^pV)7
z;*dv>3(Pwf?J<63X!BXJWi7Uv3E*k7uILuvlA&H0;av1yA+8rznQXw)yRYx80!q$!
z>SkVDwfKomi*C}-vl@tN!2FF41Z-Jh{8DGD6k|oWuvLUSsV)yF|3RGN?Qo83t`J6(
zv)F61e^VJ{c!09V*Ts`%N-SP_uOprkL{9jR=FyAQA!yACHxAcRnEGQk&cw+)y7W(-
zzTNuA#v+L;Rm%<vGf($+$1fE$@6oqp5qz&4cfyph@kfPQJetZGX;SCRxhAN&-Ce`e
zRs2|hE3NH>dj8MKI24FGoOcOulO;pW#l}KAZxk1DQbK3AOJIw8z6iGPI3O2cxRDRe
zt1&DzHz&86I+}5F8P5(T($IQWXg1u@fpzOinR&(+|5xit7rulL=QB|6?Ueg47RD8Y
z!g0NU6|0GG6_85WV!iy4Ii@Q%$pl*Sdh9WQn63wlgUs@xSCD%SJN;U6eXznrFf$UZ
z-{jR}iU8Va7<Ey%!p;pN1XOvV=`zU4GB`L^Bqf94wYVgI%jb=M66w6ORit6DvK3~2
zqq*xJNoT7Gbi@WHUYuq&9nl!Kmf3`n{zLlN1(h0mFn%-%z!j7=pXSts1+_+4MK9lc
zx)z9PgN}6X$up&Zm0-*e;1t@pb}Bu`m4cjeO-@uKm>oBz_-~-X#y!pz;K>r-5b?A~
z`)kwd8gdkq*63!utuHPQa{`wLW7D-koMp13VWt}r<kT3ham+eJ4BP}I<?U^PT&NDF
zLtbF$jVbGbIq4DHbWqHrg*fTyx*x}ieUYHnk7<Y2jLXX(F~FFHZOLEOBbGa;O%UX$
ziRNmoUnGdv`E(Tfq#!4VJ=hSY6z%b;Fb4CQ<Ku-52)e&$u8QFep<T5e4udZc>Gs#s
z?@{_sfvyOhQWE~yjX+!xiqWnu;O>Il_?&F9&@HtwK$41i2{?8ZMMq4#fqg~zcl1oM
zSdQa!Y$Tm(Uu?7q5pu6)wu-Gvx>GaR<)U3o-7WVH?@57V!Et9j-V@OpG!w?I9^VR}
zo3*1<m~&IY6phSygJL03&J6n)WcHHg1PoVXqRr7H1xnt#a<d|Z_K|o<l$%F)1M>?q
zcor2-iP(=LD~oW6WI4M|%)2=NCF(=Xx1z$i8G46f?@ZdGVeBW8bhtT%x@ijtPeIx(
zaTY8=@4FJ^&RPRU8k%2YOXB91Lxjh4z9Ga#^#y=Bul^x--7K8K;1NGw%3+39oUv7r
z+tfNYO*P!u;R$uRRjf!^-ZtGD#I@rkJ=U|X3`jvF8;UqofHj#&2+ZZ&I!S}~4Sg_w
zIhc|E%&wSqn<4||B`ui?3ULOS%&PHh83l&p#3()%qM3v<=yBT?3CQ&uBW*~C6X0e7
zv(pCxB&m(k(c_lwpwvOJ+>J*>@QL*t-mJcT@h2}9Qk97UTw)EPB(_(C@jp9jJWuQZ
zVg4kSHqBU3h`T(yHJpD|VlP3AN*vU_R(`ccyb}BG$XDmVhQxWCEUdNA;#n5{As8;Q
z(@aTo(TP4%7v#DQrAw)&{{n#0>FPrKSA;wKAX9M6w{sDj69g;(Y62*}9fe3hn<fO-
zVuu{3E9{c7G1mEhGtMtD?gOLo!?XJ(!ki6zo4(j;mx3NQW$aq9e@XMb8mEc&80bS3
zm?6@2VA&hDdi+m-S%|VD-7dauS0IX_8;I1COG)i<z;0j^b=>OX<2;3(1~MW$gLegZ
z<ahMMQoB1Ujv`{R!s3t|ZJZ*)qa<ze^vtHl)11Eqx3YkFr?3saSg>n~1lx$WMZ5VC
zj?v4;J3{h=$yP^doo)62a@qR&(V9h<0P|I5h+0}=KVhEeFr~m0L1y633Z$s`OsL0m
zBB+sv@vVEB;#>(m%RGCT>c<`(mANRVPo=Eg>JaAilLz1AmF#ALG=wheFiu_&N$oH(
z!J|SgcGw%*wZrw8u0tLpq*Y2?>yvbTia<))Sz+=!Qv{9Dfo}c<g<WwpPPjaHSAe;#
znE`KOo*2FrZnpLgb6|4Lub`<aGY8}O?0um~f10eNyW1d~rkpr5W1e;ZcczJBSBzCf
zn*TZ??m^$NgCL5;72f!Lj-4G8-zd_Y(ykvRKaR$#U><w)`ev7{F%A^!bh+}$G|$7W
zHK#ks#_V#1%@}sGNWr@3G)G7A53Wj+@8*RBxu)Ej;e1gvbz}EXi5*0^Zal^aGpCiY
z*&tQoGSMzX9~XqUc@Q7w1ZWJrA$9WI)oDZ7(j(2^rX_|{CFc}_aj>vl8LB7=?%gsP
zk3VSB#9?3;(OfAWOw)~az!e0#22-Wkyg{_nqR>qg!0|#ID0K)#s`a`d%uduj$pZta
z>>)IFEoz3Q=4v5Mx7mxe>pnfuPM3!<(5Z9bUXTYR=|~aEf;MW{X<@^x5qDT`krors
zxOhdh8H>YIEcO=b1E7O(*MUiVb5Uf?twTJ6U`Z^xm0%jedo@7lLZkSqJzZgc<v9Yy
zzgUQiiaRiF0w5um5XakiwW-CMg7T=x%}dh2F53^r%Fx{&YbfZ?=plLVk3HS}!eAuB
zAAb=8CYWV-uR(-s#0CHdgFEN1@(CPs?#EXbkpZM4o-M@O?c`=coG;o%<}MFM_oMMg
zLB&|8p(k&|O~NQZa%%JAK?s;8KQ{OP7UdG4ePgt5&~#kpn?_tE!s#P|r}u526y$-5
z<P;kZK~kcg{ASl?nMPC_(0=x4(PW)0<c}5Mva#WyLKNV4GyOnsi1S1{%kBz~tH7lj
zN~n+BK%MpLIYI7@40gTPr|EwCVNlA}jw38qmS2jbrra`R6}?1M>lYO2XyEu|!Kt8>
z4|PX->>$GIZL6TPJ6tpkkezkI$=igubkcRmGUkq)Lf24VZ~S?L@AR-XvcFwnHy`tY
z2Hp3jfw%~8lR=y=CT<~gra2v5Aoj2VT@-xU48^^ob4xH}G(7Gb1@buJNDQ9};&GLG
z5CuHY9Hq&b=P1qog7d7DOv*1V0qn->J@Nhhi!AhZrLLN6ASgE&*K&|X*ntT1>@S^0
zqm#v4J?X6ScmC}|qaF1Fi{B;o!{VVhO_(c^-aBN1y+DxKfL&>PBqCS5A2rxlWg6M_
zc{Aq!HF0QImIAgaD~ocEtW+w|SCRw$wK!EIiPMvKn(8-2;v8SIj)*29x)7u5KcZYT
zY;!c^;s*x-xeXcVHB@Bt3ZaE%WVorqc~Tsu3vy!-M*ODy2U%067&C;JjnWWcZ^A2(
z!pICS*=ZZek536=8X3E0{OMqV<kE-j|E`A+rsXv04&lx$wm8%PV~i=DF}t?_>d02V
zFBU!wB6laQitBi<J$4tETc0}$y-^k5qE~u(j|eyI>;W$jYUXo2b3*KRI5hl8?p|Z|
zBcaG<I_E&Kh=@W6=%^z@rrp6<;m)avYrAG_eiVSascu<OoMiPf=4fCuTTejhWxSOI
zWp?6HE^iZmT1KRkqnN@HL=Xj`a`osDVdmOU6#?f2VJ-wq1G}4t1-N1o=i(F5^ch^m
zNM(KRF^=D%J76-tSo&ChXDyM*z=a~=gLb<kai1usuG55`_`C%4E=My~JB}E6iQ`Cn
z2*nU~M=?&YOVQxqG-{%ALdYVunDcmm)LGdgE-S*Zr^<2ZEhSCkOpbqucHiXgtIo#8
zpWxrwzE$FlO!RTdGZy(RU#NiWYH`pns@2S6C&WjhU5*MpsxPLVSOAD8<bA6EGa1)>
zRjnQI{gc3)f3pi2g@e`-VE}PTA1j^ggrw<A-D87O0Ngv-_jj^BF$te_Dj=o9GdH5n
z?V{-WdN=-Qh22ZqDtLxOQu#tqZpvZqjxTW<w0jNG6ZUa6-CNp8D2Yr!H@d_bgyAap
zIMA?(aH-|MiMm}X#0gjxo>bg#l<_#39OhK-O(C9V>TH(|I}=i4gM%78RfuvTQ@Op8
zukx&PR`JUKGRJ-*&F2o5YutvOozq|j?v3q4QRXfVFRoOWsL8CFTT(IRcK~jIbl|Ej
z^^?NP9j@RrJv8HMLDW^xuf#Fu6#oo2BuVnmKTFIz2(UNLEz-eQpH{&E_>2N#-cCn7
z&k7=G)~$N1ejbFYg4&89w1EH$&&zBvsxUE;#p`j22(zsR;r8nM4aZlNxUT%hL!j}3
zNUI#Mbi}FEfb^NH5ViC9kSi~RuMAsm7iNy@QLrJd?ot02h35Vx6P={WIP*e47nTDj
zwnR4waIB<TNXohAOHeBfvn08EdJ!?zcUa_Nu8SdPa;_>c3(Y4gPqe;GNBmO-x_2@t
z+TxlrQuZH*^zyPmc-5%k5aklWZbS7xrl0sgkkdfR+Y<wqLc8|JH66*q?@D3*ja30B
zmi=+<Owrh1BSLe`BaE6iN3jBsR_h%BPJ5D`Dp~q6M_^{-6d6Y&8whX&l;wSKp=cN|
zDuL+KD<C|0xa6;$&&5LcELWH@$CVJ~h`ih4^or$qPl5iY8xwjaj=hEC9)UN7`L2S5
z8%Qg$^wosDsvt4QoYK?ADZ-ry%KiaN+s+W;Hsy{3wzVG!pd?UZW9#4ht9nMd%yEAP
zsHZGUa20b6G2DIpFlADmpDmO`Fr~$Zk_hvi^$kTTGyA!J0CN@D`DuA2;`govbg4Pn
zXOYtui&bQG;jJ=`pSXJ5SOTaDS%#kx;AX^xI8BDH2{LyYcx-)Lq?ZzQ-<=~mIcCcI
zvP`MT?zR%6*A*!-vU*2(;Nk*7Y8yF2n0h^gE2Gm}Cg0IGRG4`n^$3a}nF>w10o1%^
zLe<u+TS)GJNU|8p;N-T9MQWNyj}Vf(t&SX?5l!}d>yOX>XkxH}x8&z;gm4vk%qE_^
z36f!@>m#*T>So83uhUpdgxRfUTWw@8B+zW_t8}oZ8WCVFGC6UBn8zl|A7w*}luGA6
zQJ`yv;)Ko>X9{u6IO-hiXX=kjgt>IcJ@I8p%HH?JTlh+05#(rw*>8n%QJOU@NMlX`
z{uw`gafwLhs3D0A@PrW8h<yvD4BBcxH4}(TZ5P@v3N$~N4J+}g=vH2sPQ>d1T=?Wx
z9+5S^yA6!|HG?6oo^l7I^XKSR9p^zI?oWDd2T?uU+L>}EFxBGPHsZp$qWl~3fSmR$
zcNd83HNbt#jYZP7XwN#MQ-sUWkJqIEos+cv#SyqC-V^(aaGiBN;>P%mz}y(gT~wSY
z#Oa_=^Va%BVQyOa|CTO)vAcm<0%bcF4y@x|MX=e0*VJ0<B${cZ7oU0?-b2`|<H;}<
zlzjwH{vHW~XNqu1I+#hR(d8wALU3qwFs{3oI4%b}aqJ_wp(QBKlzJh4*87WOFn`4(
zV|-7D>pddjotp%_$}G@j;h3I+z9azFOEHEW-UCoA(NUIF_QSR1gTOg|PE56kRRuYJ
zgsn<EA=<g6XC*nq8jTMH!Ue3`qWV|DZbK=}FDgl<x42HEf5(<p#yiXfHwrXsFxwSt
zo)$uB<*o=9X%7`YRd8!XuVgyhRbZZ4dCP~6sAhXzD7kaCChu3DW|9ecTD=~NJj_?u
z4?%`|QX}zWL9QQ%t-W!cXm=020h-$29$_wYL!McZjJeCB1&Cb5Q<KtWHv~EZ7Fh~}
zxFN_bfe{;X=pv65aWKJ<qqyi6lm}fqPa#CFfZR#DQ0GgBvCiXQFnPk{v2nk`t~|0E
zu0`4SHsf=lEy<=h(1e}aKfwoAfMbtA-+4=**ApzkbT`?n;4X^nyk+h@DD$9y{te6p
z#BiJ~>&;IB<etKmr0vVLPZj^kS}$GfU-)S-+K~+plkPX7s6n&V5vM;xICm2ynA=3s
z40y{-Hjpd+9XdCI{DY|*>@F-f1Cm&}GZaGtT@ciM+EH^C0*GPPkRk2Jvk)#h$2+LT
z9~O|ShZz_%Y&GV3&VO(`R*79in70kRZ`Gz}lj1O;E^pUxJ#H51Zi972CGN^8px>><
z^CB{H*!k%sY~|;HoGaVEv=@)9g=u<_3FjV};07f2qas{rylrD@wZ#hsz_G`K)niuy
zt+Q)&0pu7C;mB#;7x_X#En~-Ykz~SwStBOD1mRNf6c^1+-*~pfSk!z%__;6|svVP}
z#a|}u!AbYk0=JY9O6S*jndgoa;GTrmXh`N5UAo+K-YbsC=#wA2W;`gsX>r;)84pB@
zz6z3upQg9<MZ1KYI(c^9S{PNJaqF>{h+G_uTPe<puYtH_MsNi;UL;+POc8v4Ez0RJ
z$7*ij_MkA*<LS@-crr(4nSvSHzCKg*PTh`f#3Lovmb@}a=6Zv8E=840v8vu$2&K^C
z&I0_4GJ@<NP%-T$q%~w&M74cu#%Ds^Re1IdXBn0F(VIouEFv;l+D(Wn;p17h03QpZ
znti-KG#GQe1!Xo)#Y#U`7VQqim_SV0S%C9lN@G-4<H!F1a;lV>m#NqlZ!0WM6tcZZ
z&)aS%P+h2w>C?_4oHA+_8vi5#&YD$kDCU3LvC)U3hZEu0xVNL&Zwa8-EE^R2r|&@J
zfijGuljPPEln0hxwBAItIn8$|&@L-u;1F&g{vetplV|gIx%`U54K}~;@|6;#ZK#xO
z8wkifjN3Hyp;~M&%xxp3gYKP@(XK*Wl>Q1AZ;uk~(&`4ArkWE3xqb+0h}zc+aAzDq
zt2X^TNCpypJ(m7w@k#e6Hx&t2`*5QZ11&<2k?t=dH>Qq2cYYt*nY5+Jw<>^Cr*n(+
zG=*Iq^=|Bo^@Iahs}TM1evYoE8K(S;PfnYgBIDvD(G~=xrDAKC)|$rzo9mc*YMS^)
zh<ip7ztV-)1wH^IqcnJGF-c@=<{V-sOKbT_LS1~^CURu^iU22-oYJPN8asbj#F9-E
zcEti*F_t%s&}qBph2{Q&(_#rauL^QqD|FBI%NR0+lZ&zWJ|afyzKAm?iJ<<PGPOZ_
zL9|np3aVKdiLoC8xfVkxQM3E^XN9UI$4-hJJ|XN1V04Hl7(Hsbhrm|9LS-x`Ex!{+
z*4Vxb$BktKt*b1V96o(Y1oNheeGsp}d<Nk1aD#(0OU|GET|kI;^(t>3>{UV-tg*Wp
zia!gYVtPeGO1{5|hA$k(YNQQ+4#^<lK$vXL6-8y*I6Le84`G*Ej&d|D#k&9X&&;6%
z7|cH>(w(Lqoqc>Qn#7pw(Wi8NQG7+<>yL9plemm4&s7)(sBgs|GokCHanP51am^8&
zSS%L!3cy{i4o{gwR~16G7~-m2v88A=0Rm9B!fpbDc={g?yT@Sufns+twBu0GxfkR5
zfIY%vLMV^o$EN=)K652X?+INh#D&r&$!1LX8UQYGrl3LaOOgKG#l;G4ObT#cVQT82
zlrlWKzC_z<Jmk#wmYMC3LS35n!BH-W$CBTGP@0B3f9)coRaj}7;%Fi63pIpsy+QoP
zG7>_k+IjTbBGLf+0oh^QDkuy0Y&NB_Q>o_p4mkDT@$oZ-$vGwQ7ZI)%YZYq2t!DXo
zD`O7Jum<PdC_$Ms71@`ZB!Cpgt&M)?LJ{smXkX~tC|RzWwfIB68Zl51K(CSXiDGRw
zDEFx<3Xb@?q}#P$S?_y%&+O14Irer!%xlE7YE6DgslQNX&8jBdZN^&TQ!l19(4-!u
zh%_atIJTqli|<2H{`4;B9wM9w-q`Kk^Cz=|xyDlVtRCY9I1<aERseL-KX59tcZ=`l
zfN-`bIwX<N>h(DRT@Sqi19G$w*MpkN9hK^_#av}PO2~w=<J=(5Pwx!E^ky6<3?8B;
zLJIkZD5u?98PQy~<PSmE0U%y@TVR>K8SBhbMCIwlHZ}{}2yycLI6h5^sojLZ6b^G3
zO(NX=u#!nH9-l49jfO=p@5qX#OHIVyck6i_9%RwIT4BfIqG?}zEZURw`0-<7;*TKB
zTExKd>%?>c{*JGWSYSR#*Ne6j&#s#vgp80<;sJ%7I49rOU@TD3J-TXqh3GsrNb7)$
zkB+!_LBQO2T-@a*ZfCqIEO#-kG$CBBwGfC4Hh|fFC4MWwQP@xMT2L&%a1o`epJd{9
z0&-nzG|^k4%{FFud0`KnUbmnQgl<h7*9vlZWbihm0dkYDTx}gPF^LOtRd5N5!bX$O
zR{~u~bURF?S1bZRt5q6IYI7_K0jEY8F$XB@O#%)GxCKAhp8O&;;|P7>7an!Md8P;_
zi2o;Yct?PnV+6Ci_eE;$!1x`D#WHr=C<l)*VKF|tkenR$!8W?{Jp!F(ds5%@EdfA|
z>3Y*;BH$VFG%vs5_)D~hggk0%K+wIf6KZ~7$fw8Ye!istlW3vEe>c&bWw!Oi2@1Lt
zeZ$u0V%Q}RW*}F`7Ye&wFv`&hX?$D$!}#szIs>cD8%q(-m2York3E-$bVKR%KAkG|
zl$ajXO-sYEKPFyU6M5())?NlmO@gIRJR>5{<M}y=ZI%V1YRyp=`E+uAix8^P*%9;n
z#6L@a8V`zaan>HYVl<Y6G+*}{W?77X3Lqz>fNCtaJcK#M>?j#_h7gLyWgA{ee?%0K
z+34|D?FtZ1xef334~V2qP+TW}`dh6C?VNPOmdms=0qMTO*#caz^tv?6+jk|W$>fOQ
zAod+oq`}QXn)E>d{-qn2%qZUDD?whZco|k+{dDIf=3Ke_m4Vk2V+7Dgk|6JMK25Je
zsSzH#D)EO*pp55;UxYWMx*pFgQs-huXZ%OFi-1;?WxpCHtl~tb>x^yoRUtf{u?LzG
z3yUTPSWu0<M3}+J`RY&%2{D`4nd1fN?*g1Y9hvJEeX-zbK<?gh$RV9GyjonS3&?Vj
zZmlI>L{o&j$D!mLOwK~rD^tiGxHa21XNrjuLdvvwNSMcg%!`;mnd|3Y9Z>T$@0V|`
za2}b%gB+cY`6)nV2(NyK`R599BKRCaW3P?VVru~AiH>8w7$@2ZVL*lze*(-LOi^T)
zGf7ceZOinNPwx+yDJ8ajaf)b{Af3lhf;-E^dCL^-(lavwh!|MEuUQm_wXQ#&7oCfP
zLZuO#k2Szhw_W_);{fRJ01ckAK0%}_DI;d;O$R++V9}iT1YK=>0q{gH)7?V?;AM5X
z?rIPw#X2tvvvYq)Bwd9Qm3llT!j+OGryNJGGZBcv$bBKY?kl3qE}l@(sp;~QK-}_N
z>+MYjG{*>YG8h6eF(QizazoTHs)!dvyUfhc)O>--0Ir$rVYD$Fjm=9CJ8;><(Lh@Y
zbdJa->=j5pVQw@OFiB~yW1IxLxU%-)=QH&)wU>bohoZ|&fu#0&|2Q@g(X#9A8^|*<
zwVtLAu1_-4U__oS2#)p-48($KJCPxEpG8Gz;i;>it|ThgiWZe)WVR6kogCdXox`3c
zC>PX{)O%$lb{0+9Vl2N7kzDakc8ST<U{7J@qt5to0V5!{5Wc)<{KM8QqR2z!P>fx#
zh+=c5SVxGbRUT@IIoF4DskKw2>#Csz%9Ib!Itsc1Sh}$XL>nx~yeqBoC3E-cLzW!t
zJn*jjQtdG!6!z)<-?$A3=PKacy&3xopiy+CE|%O70)DbTVsCc4D0O7~`+cpjYwBJW
zV>SYDxoz-*$QcI+BsEURx?;k{#Xq=d#vUQ&`I$eXPHM)gnZSIpKh_iB^as)|CC)4n
zG|JiwZ?*|>a_`rqqtow`g}D@{lr^8C`9D=4)$l_&XOvNpx9d#|l1JlQfo35Vs<LHm
zHj{*Vtv*m`G@bF-&x^!q8XbzWVBWTA0j8JM(zWyV1oGD?*Dzx<!telXN+li^;WF!O
zcLpt<B{p}2_EENYXN%xZtU6<@EeO*y#@naE2_mTmyA$S)8KT^BLs)GL#bW~KBOTLX
zgRKa9z;nfNFn(XsT{V`*Cq+{vW}%)~e(Oxg_UB{~=DVE7v8(v6kjzmZo51?vZ2;W~
zk=IA$cJl~96ugPe-lqyX6<c`llmZAjqO$0TW+p0C)a$VwU)=qBxU0OFNEbb+kp@JQ
zY^B;2=a=7jIuVx!`P*Q<JKj~84A=+P<0}z)NKNaHEw=Z!9xd0~iEz=`C30rg8<Teg
zauwU9FH3{__7cP>Gzr|zxKALtq?-n9JDK3pHTHcXTxIrX-AKU;|H7Y{PzU0Wogthn
z$4+=V;yS({FX!@%u8l|Ptqbl7?&*|XhCJw(a1-o8((!W7PGFvnTe|o$`oA^kIkA-J
zT8(Xmny>io8z0Y$cK6{f6Y8@=b_1Yy`R2@}qTnDJG08x4?hfhV*Ssx=g#_hn{F0d>
zj`?K~F<JhwJX|9Pp5Yy(7W3=@;bOo9?mUVnS9zVph*OlAh_SUioc(N1kX+2P)#wvI
zYM1~F#Umoj6Wx6Gqih{}6^YQm8lpo&TvV8x-fPvWSjOw&_C9wm|0cwlaTvlT@VOE|
zzXX#O-kTV?cj9U$4j1h|5S+M#aLum@pff%C_^JSR#mdk?d{KVqScjoG%RYS1&B}Qb
zGa(b!0VT*872=;>3p+<3BcYeWu=^Eudmyyc1`ydE7X&BM$!5G)Mv&+$Rlscf62Z)s
zlUIiJCPJJFW~l5uxqwwEAY5)}$%-yP{tqcY4~YJ!jMRa{P}$fFwGqiX$K>*aU0U8b
zp06*?guSC)!i=i~(S`_kwfHI%bvZt&wEK%r>*5j-&X`7%W&N_%A}!|oq^JC04G6Au
z;eT}ng>zQ*X=&%Ro*)g_E*7oNGQonv%Gb#k_4lN@-$)ekD(uf<nh1};^px~8OpfTC
zBVEAmMxCR54JL|udo!*R>LG}AmS0%7RTzcppoib7Fe$T3>WS;>CV1`>SK!?cZu-JR
zFrE<T<gAfleya48m@bBCpmhpkwx8OF(|ZYeBB2?vHLUtx_DCN%Id|dqB$jPJBt9V)
zDC|5~HLHlJJ#lS`)ubuch0m9Txq!U7i!Ud&*DC@^wTXNFn6n=u4_FydVXG^Ie8!{p
zzFc9CmcDl8$oECNpq)~lH?+c1zxf9ok3737^snVayB#^BNek*~CB%Y;DNQ5R&wnAD
zptRy{sAzJY&Y9l93cCeS2D3@Dus(J_V7DNrFQDn{qCV&!$BkV(rYlVCc=rIgK}4Q>
zboV31HUVIIFIy=ET}CY9vHe4NFDNrZ%WN|?8Y&R2JxoQj4HpPIIIO^8DIu*)@V68j
z37{T09*%7lrjz39knP(6nZhW5hB!@Bu}+{<aI0@Q71*WV+*t1g?>7SCX5hirdORxH
z&ClvUGtdMZO#>q%wA3gdwiDHw2=PT42MX{n8j8u&=F2697>el$)5kue#4}gg`bXR4
z5bKE&q6a1a(N79;fpif#)|?JajioKxS>fD7)6x>6-hPfySBlF*O{A(N_Xlx_<QITQ
z&1e^vi;WfBP#ji9^0iZnIOhSykFc@IoptP?gt<U%Se3^6qG{*V#>Mb~1U-NEq2D`R
zWbVS+;4>rAwVKiVHMp}z`OA6x#R5oT7(X+=A?PY)`wdlTrGr3Qqd%QjurvF23E~&7
zHmDP(IO$+wnyKtNlC$2^OK7SdXkeO75KKg`6Cdld>%%PA5nqXRB?iztFLFrnSw10R
zo09IUjP}Q|qCJ_XEr{MJu{F~>`r>LL_cf-)!G{(p<4ig(7n!>n7fa)t!~8iJh~mQM
za7cPtC%aqrf1)Vyxbc%>_Z%KV1pMt0;EciaD&|cha%YvR&KjEeCyoR&Lu4SMo$_f%
z0eLXs2UfR{Il<sfLBWnWfL(?7zUt9Mf~>5}@BIDKV~f8%F`>U=!oNeX1_j=d`eWDQ
zpz=zMkja~AaJ?#!vT~+`1Iy!!7z0ub)?`HYjX+q^%ZV`8=wqQ1pmSH|wmnMm#f6ZK
z%d)d5SC|<UGbL8nx-eHm-ow&}5Y*~aKC*zN*9vV7emYMjrvA1-7=H4|cA^jxvxA~^
z{<O@AhL}p(Kr%|Jv9X{$Bw$9&aT2s!9%r>|6hna7!K#<vKABX;!X%h+Q5BFI0G4W~
zjFfRkNVzL+7T~N<piPR$%764|k?aRIJeeqFip*q_Il{TZTJvA73~f30DZnnM3?}-c
zUqGt}_M!e-j>AoO@e?<FrxM3}L$3oIzZc@ah6gmep-(ss$YURCTAusNR>0$g(|#yh
zFf9J*>83h+M+ds%GEq5m?zpPuJ{H7al;ty582>BMS#u*)``XXW0C8p0^Ca?PHttL?
z`k7w3k4a~l&@oqBE+Y4*;pxLY^|<|PkYZ-hlb0m=```H=REg8$I}v$4sCC9>=a}q5
zp~4z9Akr!Fz^F9#8=qUmAH=oJwj#}9e3fw!vA+OHM{gdC7e%-^(evOaIDbr{_>wb<
zU)$=;zp;q`7dA<*$z1+;fiA&xj6vD$E_+__AC^KmUAkL<89M}{<t6W*=g*W>qr>(7
zRG^tDU2&hRVHdan%++u3gvVr&&JF2FGj6vKCr(RqIfXY-F9f2IQN=MJe|Zs<IlzI`
zgmvO@0nR2repQPvguxXK>>6?S#l;`Ije|m|a!CR3P8jw+*9kD+no<>fDVpCg=&r_Y
zmzuyFczWz3!UdL(HN=*yg?Kg?ZgBR2H7Oc_wJs~-NvSe08b1@_ipdYGZoV}xFCyt&
z71N(Hg-}%=GF>CeRZaaf{wu&$)iGmjly&R)E5MvC(+8J+c%|XWBATBhqZVrmQ`grf
zyB3>?^az$IG1Evo8QEK~6XOCw>R>ktvU(<YQ$eojP0yFyqYq}q2yc5aymq+?gjHn_
ze;DlwXO_w!;B?V0`hd*Y`Wac1uP#!+2v{5QS$|)|!c&p9+RZpzn3u!>@`k&NUR>o2
z`AcgWN-km8V{+p~KeXhS#AzL=A9qcG%GzW|0x<4`gt`)3F4YixMu-clr$r<z$J2i(
z{*_H@t*K=$uDQ1Ox2-ZwmK$FQaf9i)O*PiK4!}dK^~B_8Y%P%9#iLwYOA?`#c<jXJ
zzrOgL(@|`74=I6OCgagKS%8bnT@qZ-#fL&Xpt^9ZIq?Q3*N&a$#yQ->O}YmZHc#}v
zGuu*m_j*>Sr&VNiHdVacAzmfi#YPFz$t4X2UIHoh2<noU^N;-Lj=~)+j5=zuh9JIk
zDp`qM5p?A4njRdvMU;!p%OJV{xzLRubdO|RPK>QYn{zm*Vp*6bfO>V+I8hjhJ48AK
zPM&nt@?#-nhn@k81QC+J)YJduFvdV}>P>`Q%;AA5t6($E66Bn!H65-$CcqqQvJakc
zvq?M&sp$_E>3)nY?ofQ5KeJv><K*B!B0V~inVfh0wA7Nf@WBPfW`@2NhYN8^xRt8J
zynlwK0%-EPVjmGX8=h~EF}Fgy1$2O#j)P-kfz%Qmd+evMf9S37*w7oIT~nqPnh2|b
z+h&5KZmq?vw*$FzvuvZarCoOw=1zbGNm9`yuOu_!ob*USV)w@=?@)YqrQ(>V8B>K)
z8>xcgz)WC{+Z$u=;7jhaIF{j##nwWohjg56cltm4DOO@Ygp(#q<mWbJc<^10LI*|A
z6XAN|e77GPZ$qvW2P2r7vZ*>s(ab;29@_k*<^q2KbV;mqiKT=%wa$JXT;nm*?FE?~
z3^_Xq)R?%k{8`56bd8x11S9Su#*2#MTqV8tVkyAZ=x#uFI2IRxsRA5}8MTQ*p3}+`
z1iB=g(dZgT%(w^0P25#s?Yu!WESBTxSmfR!5lr)SD}X=h0$p};W=7Al_^@y%E=Qu8
zX>S(fUf;o`=*P<#5-G*@uS<xQN5O!-=zaX;s>oRe{%Hj`VOiHn7ryQNK#Y8OOT?p(
zs5}hdA{vGj9sqG0YtCWH)l9UZP`4pYx<`@|h2u+%O@Oh>g=x@8nYG%3#ZUNO$F>`z
z<Zb^72GdAe;#Yhs+9hn0LvTzj=6ndqC6M!M_%A!UgnBW-hEcLeU0yO1OoqH%KNS0w
zf3no(_h@-3g7n;;O>|#eb7Mds^)Mit&^0H-I|{m0^l2E^LFq8)bK$vTaAB_&-wJ3g
z%IuDF>&^BEFh#(8Ywh@n2-?j`l-O9L(~v-o;hg}+7T#y#r;ieL!FYSB$IW_t3DRWf
zFJ;<1^qBwZ=-{H1!f+5TD|{P`D<6j@d%AO5ypsvN9~d8rFmD;FY?y{(g(rYq9(KbR
z$nGe>%pPtajc^)8@2d%Qx6o}_M6jbu1o=So<r;q@j=MjLa3j_c?SI(%>rK$}gtXE|
z+uVsY80xtvLEV*5;|-*9(!+%LU-~f@ui{%Fq|Vb7>9FJBr-~RDk}93mo(6I4da#{j
zG{po#t{tyrA@84A#=%1@?ue@~_8H>9t`VKP?Ipr#;2o7V#!>g*fyfvSa}=)@&xt0!
ztXg8mvk)%3TZ}<{)^lLF-t1$@;sYW4(xE}ncpk!?M*3(pT`Y6VCxUaoXNgpW?h#~m
zHSJ|Z=;!7C$uTFRLDSvWg3bLdxgK2lMQDaKW<dP3o+y_|mqanW*iK06Ae1R2`v${3
ze*O_(M_5i?ktq&f;pfAdVliD|?U?iuNl>kB*)?4#A`dDwyG-EBfGfWY=o;uiF>eA_
z5z2p5vL4%sV0`2YG7sc$JYL(3C0^m1d6LfBffjoe$SsPYFVes+0!WN=6@0CU$h^fc
zvomQ;aa<(Wb!D1qpeNvvT}<2kH9oYM#(~!-qL~DGkd0q2J`HMAT`1ZaORL(9Zv>Dr
z%8N>jdBd?W%H~3Rd-RDmzbkltWh%N=h*`nls7=Q^I~tGO-vo7mrNWj)itI1;7wq3h
zv_Rs;@huRhiSd)xihXlfK6LvOcCt8p)HJLsMYsI})P=<yiPh>w0p_G+a<+xB;M+iM
zFq~IQ*+?z#5J;tDnG(;4aOX}-oI27f?-UqPxgW5u3d@70GJrQJ&Nu|+nGEqpXG_(%
zRG_)dOw}9nz6<Tjb2E^wBU=R|u{8y|@(dVd+9AY0cFN=L?C(M6ev!{MHWcI<PvuCF
z$3+CVdFpYI2p6Zf(!rS23EHOuT|KGxX~uz=`=4O2xeX`Ai!1D((qkIvu#kJ-FF?t8
z+-F20n=VjXqquRj2$uwfTMxdv1h_t^zHq$}#|oi7*{nG}$TY(W9Zn@W{>67nfuq5A
zOktM-p?#|6QJgY-0B8mz6&}nTBgl&uq6}WYn384P`I2z)fk$Oywhtl5m>!Ii%b^$%
z?cSrq5ScRb%M~AiI!g%&DA$=!migGfQpPr7#do49BUkEp$bqnBlg|E1TrZMO<Hk;i
zPZf6M*(Ye+{OnT@3dE6FTbwV#pKDdDK-dj@2I9=ybO{oV=8ZT>pri48qi&U4C4?*y
z`4A)GYLV_CNblTC`8PC0=;ZKYhQh6HteqQt4sGUf^wu2Klr6}aCWDOhM&)?|Nuy_U
zIG+6vVK-`5a#{140BXUpRd0;@0)n}g$()g16ID#`!#3KiMqM!3a(Ab~$&@srpMD9H
zy9!-{%a~sQkfm(*UQ*Z}HHY`aTLR1kM!syCzAO=0e=$;qzb20B)`LnEcQUcvH^pxq
zTz{ndT~H!y<F)v_^(}GCBAkujI{7<j=RL@kr`|YLKx+n-!M>(Xsed+{Wz*Im(mPX3
zf4p8fYk?tR>dx9|d{$yfQKnYiYc}Gwx~#6p3?e5mrQaspxx*Ez@TdUS9QA-?^i{tH
zl6p0by^e_7JCsZ{ZW7}9BtwSVMZ0}{Rv=#8C(zmSrHg<Mmw#b#HW)99cDh(xN?2Rr
z`yie&Y+TM<{k$-WLL1d%!`UG`pi13(Ec^pdb1&U#VKyC3KHN6a2R9c6Tby+AQk)=X
z%7i41`&B|*cy9GJMw3Uf*UM;asQtJb{85RZtkPQqOU%KKDw#YJ?kmE*63a~Sx{fp%
zcxG)S-mQ0`l9lB40?ipIA9~_A0gN`J0PP3go#{KLEVcNs1n|Vh(@tLta4oQu)i&bd
zxj@W1MZur}i~6~XSS)?oh%P-(fnaH&RytJ(Ljnig@ppx(5K~tz{w>06!8eAcZ@|^&
z1#~Vn2NNLJX2M(+c^5^9I8jJz)#YkjazDn*^dn%mhK{?G`>lc;Go2=6J<PcIfSnEJ
zZs;l4j~pnhHO+9b=VH-jq6`js_oDoXCX~na&iF)_v*d&oLxX1gS5WQ%{i8BNK5c#w
zrkA8gi~B^m-fSd?^gIm%SRh=g&~2AnfUwJ~4KY(_5-?X6tVv3Tsd0=bSjlu(i3>6z
zmtk?E2&d1UUXO{8{%i{Y=8if%eY)i4$$~uT^QvL&DAI#j#bTq((K>}+eqnG*hDBX$
zps-nj6uQ<rQ7r*pJg)V}s{+i#25#bFjew?@$?f2!i$D}ym#HDoZY>H%DL66~KX(#I
zZc}g?bcMn$GZ(ker&Qu*K^~rM+&=5_^8JcX*BFIU9s|E$43LD=MO=>K7KfxD5_)0-
z5vmg#sIL5h<(6j`ML2z&Z_}E|+<MC;oD@svVE^=ZUL*|xiQaY7C80=|DX0;9ir`Zx
zUbZXIE{bv?vr0qn#d)8l9EDf)Sd|;1%u!Y%y7t=wDB3VO#BUWg12|nnG{sN;kAciz
z)PlXShG=r{>B61BX8EHmVRfl$yE5{CRF_BP$XHo%^*$3MT#2_uz%1knZM!yE2GR`0
zCkYq5Wm&YJK;rnGZ%h;El3-*tF}}BK@edbsb^d^H_x%Fh;tX$mq9C&*%pJ|{H;|%a
zu>IFE0gMu}#o>zrsUJ26z439G8b^1`4s@6w{e%d4w9$l|T^=jMEamAb_O&wsJW_#?
zzzO9)(y$|q{8@;&&N98pa)e!ZHX2e#(@pmjsD?%K$FHR*btZOd@v4YC1)vQ>EQ+~T
z0D?CI?DpYZEFju!z`2JGhnlgVFc%!(IdTN_i~JY;s-Y_{b<ysO$RK)@9Bo^iCX~E6
zb8C<LGJ*T(TKr^1zPLqt#IEf{(_lP`B&RYPuH+w+{rAQqT<CfOy*Qeq%?0JQV$n|;
z&5AG=5=Jo!;}Id|j+B|&*7e3_V*tsClC$fMHjyyPcJ6zx3<Ym1-M9==*j4hA6Uka@
z^Hm(9%Hyt0g`Fb@Y@N6nT60wp7ge{FxX2Sn2qMXeYmJRrRwoQUIE5aK6-1Z?OzA`M
zH_=4YW%j3XL>^Baj(0`4lOW_{M)0|SJl6Vgsm%yk|EEP_^zbQhzUVwEh7onrS>uB_
z5;7r9yN2Q^(YdK5!E5RKOqkPVI>r!(ZSWdJH15CZl1nX?6Gma>>||77YQ?L}U2%^H
z_{si(J?{TRd3j*%veJkh>vw`_16|8y9$9xyNax8;f>j7XN05uAck3CVow37MFxM$5
z7<st1o!24&E$;k31GkuM93Lp*0LN>~C`^9Z_A+|c7wP<v`I5$GTS2Y-d>Ru6lo%6r
zatjm>3M17nrpzZ5cDs-&z>M(#?pk=sjekuj(&R!eeUy&9<3uobnJVi(0_Cnka$#^z
zE!%(x%SaV8Jsh8`Gl@vBxM!F@W{7Yan8nd>alQ~r)yAQ5buGf~Ryd3qh`(l<n~l1n
z7|)1q$s;Xy6V2FcGO&xmY=t>eBhD)!4z$?|55;{Yq{n4-fIW6fkso)H24Z*7W<ndY
z>R#pN3hIHlu%xBLj1|`*x+~KLL$GL=D8$W-(PeKxhncl_`v0gp>o{Ah>i<iLfPe~1
zIwhs<+@z$tJCyLuoHH}$&Y3gAoQXT5q{JmJA}L5phY0vpM7p~}Qc_Z+L5$z~vp;*D
z=l=dVuV;PE&b52(wbm9$4V$df^Kam9napjiZwF3x1nkyYvAaO0IgIB6_RSY>2;%1D
z0S6R@Se`x<Mv`rlro_}A@lCHy2I_H>sNAPI*p%Xf(juHCH!70h+?%9Kv6L`R6FvvC
ztL1Sap=M`0o=~(yUa<_A7g-5cHxx=K24$|RZ<i7c?9Q}i{BC3a-!^ShY_<vCX6;a`
zBc_XV9_qt24)7NV${QojZ91MN+0%iW7C~WWKQ|X~k{~k)ZEH;jzZBwXvyH=Rb06xD
z+E$&RKj?k3Ly9ZPr0^0{l`75cWm5DA`9fP{Gh$?ZBk5-gU5&$q(J{4m(uwTJOfwPd
z5-ir+od0QqLHXQ|^+mxKT-e9q`X+zeg!ja6^B*#3ygWydBhG4EFDlQIoLO5jx<vu7
zv@!ReDPVkuO~dLf33}Y6f?e8d$2S^J3-d6-YK~3mme6@ZO$GvL(h~(zNL<~<!}>Nq
zu^5&++$ZwC1GTaE?pFLu70~dl#9<;F6g79rSiTnGYizE~v@5iT5{s{``6riy<v*?$
zO(nDIEEGt#F#sDY4efo-*|q@K#lYyl3UCi?p=iPEuS=C>wgaU^TpUb}M?{!0SqhU(
z>?Usyoa-ccnXC>#%;Vt-VzlVr0l?*B`!y8*E9tgr)27CWJI<EGfS#c}M+oiQU+1;J
zn6(oW#b7c(TWn{D%w%~@Oxx^t1k&qSdj_c9zeU1x-p`2(cj4QmX-BbhG!EUhNPz7j
zKGvk9bhkj)Ul&T~s%amP(A?-!2-9NnphBj?5N+4pinK76fN|=?oh4Y|$|#8@;vs=V
zV`-cbEA0W{QK_eSrP;NKAZI00AJk-9vCy6chC%N@QZV|CFgPQ9+j^`oid1=Vp%EvF
zKv>*A5|j7h6Mk^%q*bx8HzW<f!3!aJBU+-JKVvandmU0j+-ftTTk$VJ<}^YVnflU}
zbL|5t)^fU7U*8UZ!xx<0Nn*R7z&zt{uc+(?1r`0XFGez*bn|9lM6)mdyNT%7-FO22
zF#y%#YNr{8iZEx`_2)ylFjW;-Yx;EJNl3i;PYRkh%Ia~6X!p+n`OD<cB?J~q8d{4@
z_JeYHy>Z9Mv>?x+ofyl`66xYm9c)=HIRGGc3%#<SI(}GUNyb#eE5h6d7U5tZ^CjND
z<`d?``fFW!URc2R781?0R@y?|A=IVjE`-@J-YyXm>!MuSwiijsrmz)1=_ml6d!Vrf
z1*o$i-s_1CszsVo;$b5<BE)q^NxUjK<{^S$y(HxEXeKayuf&rgTx(of=!OOlEW*ru
z)c+>OM<v~_iFm_W5l0SM(lR|$m^r{*Kh8jo6+j}rysRQ6m{ZCC{aAVR#`ywVOhi&i
z4)T^D^QWqJKdP}x7YGICbWn@UL^uZB7B}Vy_XN54Qa546sm5(N4i5?q#A6~{7u587
zV~%d<+`GA&qvNsueIhWYg7JFl-W%47fV>5X-i`Xhs6YqwTi`@qaXOd_jh76mozEx{
zj-hJ9-<Alz{#BPddy3@QAE&Dk-268HJsH<H?yzl3ukfxc*!9*TtA%;8AO`Hjwkffd
zzFh$6iqgWfdW&>$FsweBO8=x_w-KiaQh8PY+-1h>h<8PJe01X;eV!&HsSaU^!!a3N
z^I(!Yk@lr65iS$X-MG7IVTa%tIvG-HdgAhZAl{V7*;`Mm3-Lc}JXttBn2<Au<8)DG
zs&xAj_x~%*&Bm6cJ8tWTb{&x@s6Ra^gvp|}J9avVPuB}~^7vXiK!8(ZQIiXQ{Co_6
zInVa~I8YS)!-HWA>f3oSMGnRTq8*#@U^lvt<nZi!;-BUJNZ2J}=jpl@aa=w<dqMdQ
z8+5926r|(T#Y52WdOD^n@$4`Jg~7Ul?e>?VJXz?RqA{zB?h!}88nPY}MVQ?Txg^#7
zg&=3yZJX4sXE+9L=nlrtBHWj(?;%y6-Z&zM;fU84JB$*BCfDPX6nTG<W(4nzG~%zK
z%~!U^^q2Pq<c=tb1KBJ(2IR6QSH9gzf%*fXuy<<Pg!o+FW+zWhdsn*1&%km{LoF;G
ze-7=ABJZ8aP)Qx-OyP-7D00NN4~BA$*-pqFrdNp5>91pNaGhwF%+_Yz7&rvN8T&mO
z*2r1H9D#wt!x*B+cfM%h;MzkAVk;&dS_GV~dj46otIqWSVz(JF|6xGmb36lz$Q1If
z5|uYY>CQ1H&3iausfaz3o+OH5R%9k{nF!a4DNSx;7CfRr@aCgC(nW;0*l3FI#JXtD
zV)PiEKb5F-zh3~lJTixt0YAIFgqjPSTxd0xgL4(q4a-(vn*sq&zg=#Z);zMHIX%|m
zfRY}qU=z|I+T7qBs-)$4rXbQxud1l+&J&<2C+AkX99_gf)|qVTjun)f6P<5Xw&R4j
z&!QE=t^wcYbh{e`yL7DX)wo|YDa)xvBaS`>$`$l;^RVKUnP9FU4%C^X7d#dKj`KEO
zGZqu!-bUfk4#`@d>j^hknHN}|b;|wLaeywRcA5;6-AkldOFOSMj(6$|l1dydBJ-xZ
z#uH5gvG556l-x6s?3f@5G*U-jWcw2#@}fiwStI8Ffz%rJfLcW2IMFT$hr2P-IJu0L
z9>;`Ncw!Z3cF1)J@Bg6AT1Lf}m6rXdgpfEdTS`wLKFKsQGHX#^TzV4!Q@ms$|NU7I
zu2HA-QJC-E6J!R%KBkgYPA&j$Vps&vD*?8Un9={>mxOR7Q7DwMDV?MxkkYaP1LKE@
zZ>v+l$QwVhOgWk0xKxR6{)&HcLFn`HSh0d2`hui67$}wjIO*d`uNij;A~n{y(fCkA
zG1KCEkX|<$^PCFml5uIp<0V?46X+VEx{a)AS%q@ux)@dIH!q<US54C5+f@)#DpRn`
z3+p1CiWFjL_c97Mj6H+MY2Fe<38fbp3!V<)zRo^sFur{Tv~%s^MOn4*R)X@l;Vv6-
z=e9!J<}5u!X=ijmiM3-vT<v3e{MVU;Bd6q7=HRo6WYl{YByW}ojjZ#_E59a;i!Wos
z8L|G^1;D)_3)A@m%+X@HYsK;B6v6P&0w3-d;9y+$FbgKbizkJ$hEK;*^f!FE-duhR
z$2y{2Zys4<Z#NnXp9|z>LY#%|j^x4L6=-Hi^N!u?Iyu<?XVAE(Kx)qUsuqulQ0K34
zzF`Jl?L6o_aWaOu<IzEB9idK{6+yRI?PWN`eq0PCTgg|1y5u-0uE(tN3tI1E$Bm-R
zLoVbnJh)kav%}tA9yrzCx-KZt_Bt-#DmtykpM*LxZnbdUzxIUyE)_TYir6iL9><`P
zF!=$IMRuG`w!Wx{!K;@N^xrLn)w!*0TAcn{$7Vl(!bMyl(mhFcYs{8|al0@#1)Yt4
z-xm*-5W&#atj04Xq}#kY&m_%#aS@Sjsi)E7dxBg#nwhzLdjaO2yl=!$N*ar_O57#d
z1yVPr(FbGn5-=JQA3^A8i!e7k8NB%#a_P%J%_eq$J=E-HLa21DITlao8<t6Xlj->*
zkuFpR>;FKka(NM3qHQ|EjzU}vPBT)c<6tJt(NGdbuD{k5MIc?sq-Vo>g^?8#t31#T
zWs-9ocZG3g8Ka$v8JScwZYpC8VnV&#l|=%)Vi#|gBo5N!yI1iq?30SN^k)tf?Q-&@
zWmh`HwuE`yNP3Q14O@UAfv}@KHWugV8)j){@2InZiyMWwRTv~#4VFnTne*a?>fd>F
z8Ay}GyP`?59Z#9P*YNE;S(K@oCaxz1(r(FN>(e5<*5j{)jSt5i*&Ya%44@frmuckt
z0~+Ezfw`@c2fc~E10YwqCjVI9WH5&NsMz;f2zLRQO|b6&Nr>aJIif)SEdjI_=n3)r
zA_NN#hCR;+$lZ_myQfAUp5r>O+~YWKBCZlQ))MNF@K|<NyeBHm!aeLAbP+IK&7Jj+
zyTw3fN9--qWo36H9gstWX!n#99j+HeC3!+6=DVJ6vo`Ix)3S1%KqsIL95csXg{Y59
zH>NvQz5x=}pt4=zkp&YxdfXce{{e!u>%801t8Wi==`3lDTk!uJmQ7w?EGfbq9AWlt
z#Vi4iAg>5=d+@X%vq#cywnW;~_Wcpm6&~ZR;2M#Rmv409=05>t26B+^ljZf|Hx{v`
zb0fa1$kc&w&~|$M4`pp#9A1#gYE4{;4@Ks|IU;8^^ZmtfF($-8vj}J3ne7*F8#B&U
zT8dq}b_Uy*;SgDJMjDNu3i8n4y1yB>mO=0kgTCcxeE(*Gz)^OG@w~p>Fgj$>$M(7f
zgi5e`9gHP!g>a?wyXCcbP?)!;uvku&?UVEw;ZBPC5)Mgm&21oLglI1gxSelTiHo$l
zoE8oXa!b@uK*vl~TelVN0L^1tE9e`dJrEI4w&Ujs8AdA>xD(hVYoRrOF3o5xD=fEm
zUt?I^V?#lyt~_e0Z}S{E67C&ZWq2;jyL8s=zsup<QOmng-%d`C4k@|U1i8rUpkxjw
zbhW#Sgb|<6*sXhtm~y$qvsrPhAP);p;EX=%5$6ho*Ho+$i`@&skirRetrFLYa^@^y
zvKx=b1-Z)fTs^$fipTGB3QU`|SoeMiw*Ygog!20eftlDXr?bI)4?xq+<$-{u^IDPS
zh*aF`F~@^N3jL_;GBD;7LJ_f~uE!dgV1~q>d_ges_7g$E5<@c2pCFQi$Mq>=;zI^l
zXNGzAW+2)H&@f|ci~p=|YR=2+#Jy9L`xhO!&Mw%8^0tS;DT$0G;twLmk79(9Iv73R
zA;GQ?vMC-enf8byG3dHt2NC4SYl1Q7qkNm0NrbL;TuK-@bVx+Ig9tZ3y~$pOg}qCV
zQ^zktuZ#=;+#LpL%xXF}R<Sk!b=eB*DpEi7F<`e2u0T*>h^vKAo*o2$jmQ0sSrirm
zd{Yh?IL4e$K)9eB>$$NbsS#nGtvdRL;&hRD5bK?V(YRfRY9d9o4@JP<Zaj33#gwO@
zTxKq1+DE-l-At%+!Jfz`yhDZMg_%9CyG#N-Jb4SG8K)|c*`&g0*YHpoPn&7_I!t*|
zpcV|dT8?d=hN1$a>6jF4&p<l&4$iyDQG81n5z<T7@#|-yoGsEP?AArQ%BT@)qWkl6
zAYvK=HC__oQnAvp*=fa-fA#-8^r!gl^AKhNoesfB>OD&fb*!d#lGLRsh*C%djNnof
z4OYWY`xbpu0_{qqse{x*i1{Ut{4H&=;%6@ak|yVMJW8Z&t`<aHXkE-zoAI3&p<Nhh
zm^Zs{6IzW~LfzTeevtfaCD4iFl{b1`A_S$G*fupn-?_qrH722fcwJELI9RBs6HWbP
zFqnkbM=nf7n5VqHhy%p@=O{&p|A=s*(h_a^p!NR->W<e$b7rpBAak!_{Y`J~gfO=Z
zXXXYkYVRz>HNgulOU&zK5RTq06l3Rmogi$h5PhDaPiN7KiPwg2K)R4^)7D)t-V^O|
zNneNiNa=TfA=CxovX5g8d!vN8AXxM<GGq5Q36&AqhVl5F5So`wG@F{^M45^0&{AJH
zPY|WiRXe@-?7u@gceaDsYef9?ZBV#MS09X%Gl9ZaBhD8=6LoYq;x>J|jM&8cnYX3i
z0aNnYwf9DkNQy(HXpf(ZGT+c)W`+3bUH^@@IrPovd(Uyv|B}A&X(Dqk8fs5oeJJq#
zLUW(Azh*kz>jHB<8N&>y_*9Tt123e0-+Uh=v$Q)ef6ohZMCJp8H;q{R10XjSN3@tG
z(q+bxJ6rQz1UO;buILTRgN4A8ejW?BvJ8$tF1p6i|KLwg{tPS}^-?~WLp&+mwUb1V
z`BNk3ZK0lN*`sU3rs;9ghecr4Jz4l^P5P5iO3Jvb)7bBb9Pe1`O^L<+X*%Z_TZpEZ
zC<sSQ->%hkyn?WG5uo`M&!6$_k07$VZrqkhrMuu`aOZ&UhgLi#nn<{+AB;IZfiNqP
zveP^9VkF2MC*9VdtVGu-F(&Htj$m9T%;|CqGZv4OVWe5d=BgEc6GmP&Y1S<MDFjs+
z!`x&wee(w{U<PB72se(++vAwo5V(Zoto<{>IJdqA%l^EdL%T=u5D1zI+CFb3&_&M|
zPrb2|Fei?(M>pos#|d##9Ai<z7i}Ks<#`#_F7yQu#p{*Q{)S&dnC&touf^d4GSlh<
z9g-@aE6BN^_(My_!e4>V&fLlA)?itYq{wcq8LNnJH)4cwwLYl5<fzc`L4qF$Ivqnj
z$3u*Q3t<uN<*2j9*CJgGTxCgI6bt+dXuP(_qN;JM5Hdyt+l-4uIDy{ENIWExfzI)a
zy~uN-oWGQSd`B_%Z%4=VO+9`oB6oj^E&oN#danvK*Cm^*3^Rac!?3k)aXZr$Zz-I5
zzOOpp%PDgUF)dd85C56RdVLF5P9@L}(^+Z-xYgRGqG5S=8Gwr!dc<b}ax>~0jg9SB
zf^x~^`9x>2Z~Yfcb!OtI#6F@NDc!xnu<0elom8^sJwi|(5;S<aDQo{9*!V;z{i1Zj
z-02%zji=W!im;XGayLPq9XK}77dq2J1N#W3U09Dh;zJR+`t7_f6MN19;C5t-Ec4`p
zgt&DO(y~+6yQ1B5&K8XAVl`eALbANIRFAFaf?&ky#(bu}T?b9vTx$=;&B9y<X|i(2
z)9R8?=gqK{l=^Ok^VFgjdD8@^L(;-?gF7S^obn>`%XvU@E3G?WV!Tz-9Buky?r-wH
zYE37I14Wqm-5lt(eL75#ONmL>K-w)lB+NCJx_m9E-mE!q5m!2j+W2fBh?bS*OINgu
za;$W@m^9y7WvoFxS<du$jv(jQ;sPOg*1bU(rPNF4vHZ6n@?h0aW<SPCu=sqSWX4J2
z2UFryQFQHT_IUb;NVtG`Q5|`Ad?=dq<Qy{E=J%h(oPjoH>?GPuq}xicc8m}giMLU7
zsW=*E3v&~c?~o&nPPlo9(k!l`DzW$i1f?^zP`gT(TWbm+dwJZ7r$sm+u8B}OWtXRo
z={nyAr>``)yS-lDt}>UV=&&W-gk=_Th*55X*V1?98u$Kbid#n**)X7|#JM6|F5Ism
z5~{{;O9<^_EHy@B!orR+UA9`AWr9LgVn-3qK*F8$l;rLu#@<Q`Q7f9lC^i=oalF1g
z(vndJCuz32(eWK%PkOv@7B7gT<<L`S<o#0=@!PlxUFEwDNPptL*jc1|5Zh)J4=Gw6
zEO2~q)a6y^Fd?p~j4i~=vjuq+432UlK1Z}mpnJlHMVh@MNld6$-(!{UmFY+T$(*-p
z3F&b*iJpHY$b8Yl!R<26=7mv*iAbtfTh!lGwr0=k+p`LO4A}xNA4Oyg_fc+ToNO)3
zg+b4e3mQ#PaX=YTk3@|ktByIpUxc*LRC?K`PpGTktl=A#hCEM@t1la=*lMvNhzzsE
zp??X=&5J}kt==7r7tv&w7YB<r|C2EvXA<tNmjI-$1K86qxg_5%x)l&oGCvaP7SdwN
zhWj`nu%B64Vv%b^I$adY2++c4_=%+e>AY<8upwO<!U;>&K#Yb{Pc7phG#NI)%R)F8
z&XU|M$6-Ru?deF5yW?a5ZuPEef4nY|9LRv(!E#0PiBporz<NT=kxF|Od~Yj^%C3()
z`!0PGy{Adbo|lPXy}^CO^87<SltK&slocFFcMCh{TQ!&WhEbC)A}2#8_Gk4!iQ?B{
zAeLGY!Y#?jO0NyyT0+UgFrl6jlxxM-`2~?qlK0`5!Jqm8fa}4N<;X;sq$aOYr0jFi
zSWFXS7ITTsnv$&iI)pkh23A^*n$fvx5t4?;9w6dTp>D)9`!LyIA|Q~mwQHM(fz4`=
z?uSfujksa;A`VJmy07G9aHl{AYV^_T_WL1p?wL5GN*Y%e3UinqdSTom+MKeX-YBmj
z0BH=aHzDR+lW+3`6=P<gSpsrFc%LH96;0qC*61(w?MaKPPR?!?Gp(=|u>04Drr&C;
zR3ehyqA#_(5L!=*Ok62~X-%*Ca2qAc>>XEFrPDrBCV^`QrkHp^h&4M9lH_}Sb@pY~
z;g5W#;R$BWoh+8Try|%j!H!~Td@34_BYV(cf6WPyZk1lt<T&1w>)8VHfYeGb5Z`Ze
zEWDK}ujwLTM?YiwMSW8zt;I1k5yHvo1tFdvgbzmxBm~mR{^*?q;bP)4gZ9L3niSWX
z3``54xMqJ_hYKK!3QNn0`p$(!h=V>la*pQ(k}9XP{@8sAglmblPAw)*g?67{t1L5R
zEJ20253q$#8<lqixiCqCCADNLzCX>Wv6)Kz*j%)`hXlG<>j-c>n5c6YZBdKk1&`-|
zmIbH!Qv|_oPW?Q`9={byp*ghn#C0N^EJs|r#}@+3ch*{|lwZ28!=ozP8^h~CkQV!p
zUc}6gi_D?%-A(>q3o)P2cIZm#MzO{EfG(8gJ$bh{xrDS6<59afTae3whj9G$Us;Ah
zM9GCL4i7gVOdjj_qexmRs|#}>m}NQH$Av=N6k|1p(nT9Wn^!HiR%|>pDa6Zy$9W|c
z-fFxl$T2y#Ap-ajw0apfw8!b2!NLZpif-aYP%f<`P&mmIkQbIN2AhxhhY2-5kcCTU
z3F*k;0y95Y{k287NDvwE{8HSYZ&#8XvUPwy7v@T$;we+Z?`;g?rl2FmWRWfp;`Q!U
z{3`#S6$9TY|Ho$+<7wqhiWGH$LFIoeL}eI5<yhajdYoslnG@hb@c??0120-K!pyB|
zA99@Qa{_G>rpA;_`A>r!EkQ&U{}AaW(A3L?NitOaw_tNZTX&88xi<qso>m=>#r5e*
zceCEpmn%Y?V^gy94$<D2vgo!Khm1S70MF@ixYY1IV#@-;<yKPOxkivHi9ZWA9q*L@
zGEdg$56eiB2dn4KyA_cbKZqw#As69B;0da@UZe}0jP3^FZ6VGAM|NcKNf`v|L^Z@&
zTNA`(mFl3}J<qib5G^sC<#_)5JsE|+cl4bb8TUf?n=An~H*|x;wk<-8&`~>i<N$!;
z*7b7wIU-1RjGe|(JMc}>d+i_nha$~ccK19=A$okm>tLy6LmsUX)LX>%X?KJ`H|@9t
zTpOq(1-n`;wkBy7E+I4<SY3H4Sx{~Q`SXb{N<a_%;8I~nLU>$BaSV~TS~H=JS7knI
z#ZC(5mZ&lt-zk#8svf};raKpD3`-vql>5eyg}Sv+nv%%s^D<mjGCBKB8rh`?r{$kM
zdV>(>E|E)Lyexq1`qLw<Uz8D;HDx`u%C7wHO0Zm`2bE@<NkTJ=y3;W~w%Z-Z%#<=@
zI-MaCn|BXj7ZrmYJ^!*=iC`43LLDo_Ek+g7ftzitKsReAcPMeb=yATuo5w+W0?;KU
zwM~rI^-V2UO^~sly*Cs+r_#>#*@pW-xLh!gyF=?8Y%bWzvxfG?_4`7*Mu@l(LyyKj
zKQ_ep(M+J@^PnKj4eYgPox??XYC+~MW!k}5?k8XrMQ8VBtRc$P@2m~U>S!||Ztfm>
z>pyaEtxLl(aX<bX?=n*ON&Acg_Xnk{hzxWLjN>v*94_3r4icSJ3iKvcX)pAK{xN5{
zC6dxNz2{Sb&IC!7#0Od}7e4^h{6(#fRBtH(ULZl)?Rf#z42b}n@>fMssO<bU_Bs%n
z=D-0~RP~)g>W1{P{1117alO8Y#nOSMPIufd+Wg_|5H9oLzMq27ct}+HV!jH58LVfM
zIF<5RoG>?NjRk{dNg|3P1e5<LCb4Jhn>6seRgbmWA<TmjCI}RW*y9WfbixSC<qM0W
ztuS+-i-oE;{#M3BF3y`tQV02;KzAJOMkd6f9nk7*&FOj!!2}zkj<{Du?y;Ci;C{Rc
zK-1SXj0WOABJ&cEgwgrVb)Fiyd4dE8-`TOI5a)?+6$I3x^MHYAmE_`d-PxkC$D{l^
zmxzvZRQrJ05|ZlGsiHM*xDu?yqMfGd)O8@%6GdrJw3A$Tcai28?pCC5-W_`i8y{QT
ztTh_RedS?-T>*JlV(EEU2>Ixw!l9~6%8q+kk=N=XgvXx@o@kKRPLOL2FcQy-hSPFT
zA8(7u4M-PGiUMnNgOLlvhADqglta+ln*DY5o`(o?$uJohCf_S^9C=tBjQd5o$yntX
zE6ddZ=v9*^PKa6h_UVCQ=R<-9)cyLyu^41Xt{)cQk|WF+igTw!Q!plL)|X2}QIDi_
zyG)P&s-@RrH)#|RxopJKh-f#o#;NB2u2`%W&=te;Es_$dv8*u4$pfabeEzO7Ra~U+
zJOQvvL5YRH4+!)WRE4jbH6dNvbT#^{XfvsXsT*t=>H~3(x*%c~R*fTsxxwH!qM5On
zIa@GvC9t*SSyW-U04N35lcmauv&H2)LyDYN3eqq_$tE5YVTN)w&rsQ+9{>gp;%K-d
z{ws>GoG0jm0}#$Yx4kuOUt+lypobPWQMy+~4HP-%3BsLl4-QWH;z9u=)gc}5?+iki
zFE#X?*AdBo%@($VTZnK>8ip?YW&RiW-^93LsQ9<5-i`~F7%l)_{ABVxT!4$oNkXD_
zczA|DTCEq~95<H%SQA-abg<oZ!~yEloAFb9Qyi)H#+oxA*lUh8Vxm65nGBn;t_T-X
zLI9kg3UFgK`sm}8I3tHoo&xSE|Hf0X{2Se$|4!2UAC4Bu+4kxe0`kH?G$#GuSau93
zk3^XlXv$wjSS|(^@H1i?0jepcy@%*qjme$oK%72XP>$7=xKV&PrCY}a?iTR>{4+r3
zt0!Eg<GQ~fT8nu{)|$N{$zhm@tfg-%ZCRBz{Qnl>KHjbla>>sNx($agvD(4VPQR<(
zm0o^YM^LT<y{A8p5-?t{W~WWUE-OQEX2*ne!XZUUsIbrk)!0msCl1D{w&#7mRIKX{
zr^sEo<l%&FI~*|YxRS?>bk#CRXwJOSnYMzspAwj70*=zDMVlqo*DgsjNsi7ISv#b3
z0r#vx`VEhd48;2)T-gpx3%FuF0>BJGWlb6qF;qfaH_1s-943fv*xep~(kIQqyoCfi
z_WebXU}Fe#v$*%j0_aSV%b}wH+%x5Ejn!z2qn!l%njS{Pp&}_D(r<DSLGzBHop?q5
z*6scL$AG(G>7o6xjA)PHe&)7)MAFx=wqnsZK$O=j{GVWB%1VMn`Kg%>G*Fq&y784L
zbDFy=32@=yqsIc89aJ#QG4Ba7^AR8F{VwdYj|0;f#ffAbC&F<swB^L#9ZwuzK-~D_
zHJVr0OGrxXYOdD=xiQ#CFiTB60l=)P@&Yi=V%|C%0(<Du`<z%L&FYubPuojK4`WHj
z{+J-Mhg-3DStR9^^-aui5(F$>9}h;;^li2xXRX9$qA5~Uw<;2uzm@}}OLOQAW)Z;L
zNUDY^(u=~1K9mI7yPph1hUp#AqeReq)|nE={*q7USDC@Ib(3hyiBTwWhIm#a(Rc&6
zJAQBqgt^f|RgKKQ62jkHeX;rc6@<Bm={?Q6@u>jg_FSCci!Ecf{%}INw4v>PCPY({
z){8h>RAz$S;9K%EXs0Yc9M#xHK<>hRG-tmuNPm3>=j_V`P&C>R#dQ&8NH?b|9!QES
zP6u*>tTbst+$z#pB5_<V{w|s<=@_&h)7+N=NRMGW6i1v<ghu?x6^Ll3&&^tzH@<gf
z5d>F~&DdKs@l#^cMR=-dO5Vs-D?RcAp*fX-O1gi%QIIRz+rg7*jhOo^5DF$uGwr}*
z*$gmv(N*Rk`@R5IWu-Qz*z11{?H)3I42ypYgWr0~u@)Pg1LfhV-Hvu-QmZ&fFhy=L
z@5Pu1`b<xI%>5fab3efDg9&6AA#Q0NSU?oaY>UjM8fWPbk7?B3s~W%fW}Pj*90~Q#
zjv#>@n!#OG83y61&Pn$OaVOwWiOK87O9CifUD`qmo?HCOxtQ(ATIT^UsD^l&c4K`L
zwKd9aTN)e}3m_`5KGfo&{I3|uW5Sh~c0T{+*&WGUy6)IX7!1>uRB!xQl*`hdOk)2b
zz${@gm*vZb7Zi!0tiuKp+3BvrC=+s)Xy{urcvp;-LC4?PjvItILCyhHo}5|vLJ&G^
z)~nlJqysQ9%ILl+#Jon($+=il!C;QX4U{fN@5yAUYb7@REm5cyjm38Q7*Vb_x)D7C
zcqGFlrO1X?L?l&zt_;PMxaMiR&s<C>^P>YDk!I{FfSUGU0NkT*Co6Fb{Heyl!rVtY
z^iJ*ZqFtLo6o`=vTv#G{Lz8{SMM9hjwWHVVc?kd<Vf1KU@~p_*C_@c|x<_3KK<yaA
zbchu%gK{wtYcOm%YR@V$HgPN-ObizZ^A?tE3?|-LX!U=>JxF@FSH|2(53R*Bf<2FP
zc67utR}{J6K}9NH)mTj!m6c7D2E`_27`<Q3^DG0gn=p@kZN7AN8x-XBv;#0W7sm>O
zv7OkwUZ8I?je#X)+*ivea60?y-R4Sy(b+Lk>55ZCI8k&{Sx``bjlT)>bVTOK^$9^g
z7Mi;tk9m;%r$Wraq$La!V)LuYP+0NOEZYciJFvr>vVI&=0tjVsB>o!#Zv93DUnJkV
zx=1%&lc=B}%(=^vIbwpvgt+K>no0|QH7+iLNduJzxmw5)^M8*nu4ZPqs;!a~X|8Md
z%lYDvk_R9}!%S&<{ZZd;#}N$2ZWoz*G<tcXvBvL;|GUs&u8DMElrN5_yLr4^V2-2#
z7sp%+0P7h}ykISY+{TDL^Y{LPL(J6pKm=34N)zME>x|<Apf5JP0m8}Q0iOk8YXRne
zi{ovh3jePb<~p%rbi^B?$z>w`l)C@Gw>uU7DYzx=ik}I}Y^CopQGHn=yhx&K7WpG#
zoUi0Oow02R@H_&&wJN}=P))^oW_Bb=#@{R>(GlU`5`Q8R!x7)5Ou-XGIg=jtkGwX2
zBLD>_Y`m`TeB#IN7)oXNUTBw_90)ywI9}gQf&C(){ssQxKPOF_636S?RnK>6aoEj3
zZZavn;{K4m*MkDhQaKQzzl_EU!W<WtC6Up`CDbL)J#g~mTZ-f)sKogIY`9w>b;Lb7
ziLZ1U6wQwqgq@rS^Q_g)y+u!KDa3h`0)Dv=Je(=aLlMsbXmPQ7<9V^$b3%Ijc?}V6
zacmm7;SlYF8x?uoh+ZMCfK=l2#=@*J63?nN;w#Z`TOJw)?wBosWMMZ`2w5<NaAv_B
zg=i`QfBG;yanwd1JwxBFLNWl?uKsReZWlPkR&3T?0HmgSG;!ix(XOMMM_RTW^WP1s
zu_BfAqeM6jyx*dLexiW!Q4L>MNj#`q@!`FWC?)4O>plplt&8DWAJ1Ra;x3_P2seNz
z2h`$KK_2g2DERa};6$o2P+#;xh`a=JBdtIli~%T1uyAvF1RuHDeU}Jsxn$5WQwY^a
zeR}SPAaYOY)$<&<Iv3<+1g{BPyP*Tb_Hd$Lc!ZU-j!dzcXxe9Ja42@r_xQxeDy%(P
z94?TaHXZeMG`B@5*W}CYVH4b!aJ~BoL>?tIt%GkrS^#YGn4~@x;6kEDkW{%gc&rFh
zYYwB*vttQpN<|?|02!iLI}ocr4&ef3<s6vxJArNq7HEvINVyiBPZlZnFs<_*iU2Cg
zu02lBxB1&g*XF+bS|RT1xFD;=Bch#C(#O!_7_rAwfUts%AbYb`rjkFI_)wHPsMghF
zm$}!|U~YO0t7GnGAe}20D6ER{T_GgM64EmmGs^!p9inJ=l@OPLEdq}ms6pm@7SK60
zyulcV1%+ivvo(C|&|*NiI{}+z`9w+b+sjK>;y!kkNgHDG=Lqcrwve}SCAjHd3j{wR
z^6FX<;%08qe_HXm0JERX5#sE0;(p|LP@0WBDyG&iI1XB8%+U1aI|a}dO*C%WUgSGx
z%38y^{-}^VtJF9iGB4fy5|9~DVaJ6;pUGmwmjTJ6V~G31_eA88R~gky9<jtL1tih6
z_K$}OGVeIo)G_Pnh*`p10J$;cEIjfmh^wR<3;6?NwtlHZ^PWeokx=K`iw&lvL2oLc
z6Vzju%##VpHNuUd;(b=8*55q}|C2`3(ytMf4uMb>LEikYLydPz=F-Wc&HG^_E{%9V
zfcS`PcAhu*&J18K$2KE26Xf{qNIs%f2B9f9z2bONkQs|ziDpU4w%VHoNKHBX>C*HN
zfp8nGy{<UF41v>goE~UtzE+@_%hnEQB{ST;!rTGrP<>p@rJi<QPOJxI(3tubVO;V~
zxLxH+e=9*Q`Sd<quWnriL4*lQTTv0@!L<^y`_s!1+JNXF_ZLBK2WC4KIi8LHaS74w
z>e9p=GlfwvG(!jD`7!`km#CX^H}f}PPDV0jM*bH<V0)#z5%a(8=qz>vv9E|6RP*aV
z+>rm~u5~nC6hV=b%dUZVIseyzvT!W*4*!)hjHJ?>_L9r$k35eLVf4yeD$S+U1iLRF
zszV^XPMHAL&fbhBt%V(C3pvK2Cw3R$QtDkEZfu_~gUGicj)Th}vJ_*TcvA>vmoFAB
z2Y>voNi{Ni;Zc!htegQS@$Cn~$|+PjCcFWaLG5Jf(zMmc+W0*}x{8JW7ZM#2?0zAR
z>4BTm1-aJKlg8iK0-OoH8JJttfBzwnCS@|h&yol=79Mood7p2F7uSrfp;+SsC@02^
z6RGbgz+I}v3o|E(bd#_ap}vDWJ3U6V`#<=DBq!4EUeUJ;KoKQoV+Q!ahk%|yq_b%w
z={QO#(Z&#p-=c3yqP3zDYy1<60<+f`jBP%G$fH)y5?Z3o9dS*k05#FDV7DU9x}>dk
z;m07Z4-O1if&VE$<($wK`+veWS@g=Y&_^QjNachp4T_lO(;{|vWrSO%?+I`qTo%gZ
z{b+<RXWFaxjvMhKA#No0+o)mI;wL2}g;ee9?iJ*sv2o;h+4&iO>d(8u{qbi}PPW#Z
z-p8|*p9wJ&kZSbB(Vs)RFkZ1p<5Xd0eOiNYg<p-!zbKMo!clM7{!0+&-PtFp-=F{&
z8Ev0*yLO8pIK`bIrY(GHLXei#un3n7U0)?PD#RrlmM!cnqH`}o0irL4{{`UWk(4L@
zTNes)NhN~j)hRmW&R;trdw>{>-9$THZ?+6u;NL(_8}S`Y+?fG&8(d5c#IXV#360h?
zdw(LxMdrpwr}Yj03qtMsaC5zzzVnuo<0RSy+G#N2g`2@-gn*e=TljxqZ~~3u?pR#}
zgTvm>V!m%|J)VnwuEe53Xwr5bpy-LIqMROEiE6S#*jN}z;Lby3zf<J+&Y@EU(ZsgY
z97V)rU!4{tVdevSdoI2w2%u#0Q5R>6@UZR2CFy4+S>ch!pUzpt<#ar~FCG&BH!36b
zxNa`KT?AD*Sw-`>o=`Uz{Usk8ZV+g;PRD^RJD*$2U|LQQp3yz_nWsoG?E~#kswUK(
zM&735LeVaA*AVY(<iF8ENrIc(=jCtL7l8|{^VYYDRFho~4>2)8R_6nS<6IJ>&2o&W
zJQLN?^VFOf@0K7MTpVR|n0Qa1JB?nRV9ORC3(B2FN}Pl7A0aMsXCJbgwdRL*nbZaP
z;`aim15<;#=6(x8=1mhTQmd7Anx6=zh+VuMfyc~+OyPfo>2@Je&NDf-uEsBf<UCR6
zkCndzO*1oyy5nFGE_FW-Eb0`TActo=$FoZcLXpr?=C&>VCDI(xyYo_-8jSx5bUBl{
zJX0!zaq4$L-Cen4#?1jW`ca9o%ppMLOuDTw7eeldC73%`kgJgdgne;I8L=jhW*UQ2
zzDFpk(vB`*thz{%o1_D2(K<#L0m!EjSBWsQc^7~a{wMigy$T*%Eb5?`TJcPSh+HDP
zCkXmLP#(+pE=*(Z+X3ckrzJ^hUai<ms5#V$y9bUfY_<mkI=XCCP>kw`gN3D*pfF#T
z0i-9AE*sbSJ^|bmFa{T1q8VlVxG&m&F}}z9n?_1kd@jftbAZ(}3HZas0iCfn?A*3C
z<JuC#Nw#0&Sbd2i9H#@C1zo&$VQz)=NEZ)B2ubsi&b^B)2}K14a0C&1i*SLk=1B!Q
zNRY(joiRh7DtcomF4MQG)h?Z>&qb3c{;jwj{mxPjsbzYqzCEOSF~HbQWUdP)xUhl!
z_|jl*ReAZW#jgcWDDJ@VRk92O+4i!-++W|W4fk2n2s%=T3&LG3t_ejutt4KJ`IjxC
zbg~m-d@U%%wd9P;%fF+sPzm8^tW&$>50)d83!2QE)exTvb7ovpVa~Wbbl!qU-8pN|
zKPfnOlyOz{l~!<6dQS2qb(LtZH>@I9BxFOUKMMEeicM&m`+ahJSbw<ca!1Cj@r)3f
zrgl)=zapP`TqJ*8bN`?S&Caxk&e;^=LRI87RQ0(-n4__VBZ<04K<)r&rzp%5WjvMw
z`B7}fGs0*e+Ik>9$N}t)mG)%XY+s20dCs*$KQqjY0zK3?bn_fG;wUU^RxW~~@Sw9i
z%>T(MfG#ZOD0J7@B5${9foZ?Tq+N^ch0!-ql#G4ycN=!5UHYbg-F0rCju&Chb;zuf
z>n<Vcty*y7Tv0hQZB%tX@;hO!fEGaBf?<-IxSFG(UB}ZcgQCnUFLS)2C5UVhsrAO3
zt3$YjIo%`JkQq4)-AXu}i1`D{+FHCWnmR~&_UaG$c2SU><4lQVe2z81oDV|hVI?t5
zkZZ(|Uk$fW83^H8QfWO=5b0u?7C&2)Zxswjd~ux!$3{XWfmPpHAnpfrea75XC7|R6
zW5czJ5QsnsF-w3MM5FN<sCJN73QR+_8rSLDt<4z|v13xGdZUbn2Pm9XOQUF!bqGr7
z22odvEk!tC`4f>g&}eKY&;t+VquMT7?T*lD`UJjdOk@lkHQyHHs_0sv5(~E#QRN#u
zHW8f*KFp0yGWcJ7BB1%sdCYcTs|X`q&hUuX4wwW*j`9cjq`uuEoB(iM){1up5jeGT
z%$y9B$5jKp>6m{?0dTlx4NZVck?xk5Yq{c8s9dxetVT(N<z9iVIvaT;aH3sQmJFLL
zEHV|$1;VEmu7w)$eL+r(k*!NKhSi1w9XEO3Q(xM7T`<qjGwQ=JoJpAjG7c79kAIyz
zd*W&=CwhDg%c~DFG*Wk=G$8uzm3U5sm2%T#f|BW5>NeN<{Nt=TM*CwCk?x00`WRb=
z4TP9)t#rEHN&sBfJ0jPWfA+97tHdV)NRAV)T%>$a{*Oiq-D>U)iuCpL71P2oLd+B?
z!_&8@(pLg=6|p;r$2Wx5Sm{F0x95)_T;E|1Ns^6oFdr65YcoO9BKK|N2<+YKan{BV
zt^&&jE&Uq-uBFBrJ#fM%AWoA*UU$8{F7wqjHU)Lru;N7Gq`hkbDO);vF!A;&(0Bvl
z*GF5B6NJGA)*sm>+_4#?^JALqWpSIdIS3WvspeXYh;aD^&<N;@8wEIDEyi?Cc=AAw
zqW%?+iJ%dXoyF2y@a>+K-cezGT33)0pnvuxd%JkG42M1)nqMtL<}(X_dwi}SF6eN5
zKw`T&wgh28>l}$m`gGvB97FwBv{zcLY`bwJo8*E^Yz0i+QK-{Ne{YerOJBz*^ZL76
zLpg)w1gkwh6_UrL#CNgVHUO|jn$+<-5oRtEHQkmJ-xoG6Ku))rj3h=)YzywRaSx)o
zZ*3vs5|hxM^zG8?;k~$DG!+{`_OP9UO>A2?D%<ltUYsfpMei2iOBVSsWNyqir|4Dg
zz+dKaI-E+h7&{BhCGJ6wL&t$#ggI~YUV3AXGE7o%lP*oCFsjY%=cKryjDRK(t|)fo
zPnU{4AXDGF0^DC2HA#02-`O7uh1c@v)ED)gonmLbvl4$0;Rtj=dZ}Ch#=N_Ly08P|
zHUozVBM~|t#kw?8exYZ9U5iw@{dAwUWj`)a`4-1YMHd3^3Z>G_G-*tC?gs5Tp!E)?
z-Vxw_OOYB{Ll)c}hz9FpJG+Fw^SIJtC7I4`!ra6|%umQEAKn85wy`flHEk~lijGYY
z>F>5Tl$(d+t&I6$(McuPo}}A4QC=$0Mbp!c`0NxwLP(3*I=?5%9X&1WwPtdvvFtv;
zxuu8NyI^R4{6!eW;Q<aN=6xYNx6YVRk8?$mD8~>Ed@qV}?c4jXj%L)pCdl2SZ5m2S
zpZplwwPjDhRwX9?1jJRU%1TeXjwu3N4ix*i#X!Y%YKh6`zmDoLO_+N!CIuXPG@8^^
zV3q!I&Rq8O#Lq>$a9Qmz4lh$o8dGBCRl>|SW>Kc*dG;&f;ieXCXY?ZlIcbJRZ>)Df
z5d?8LC&*@eR6_C>!84w*!GVMtZ+w-L32^35!N?7Pp#%-BI8gw-Wqq6;oThJ=7)Bvd
zZpFER%s%WEbO-#X5E=xX77fcqDvsaJ14sYY&UfyeFq_B91voZ#EnTQ+eImrnr+FKr
zBl5Msc?Td3&Z!%-gsKU~a9dm<!YK~ox&x1OH2@cp8!~1LM5`r)BsyL3Y}-jFdI>L5
zA!<KLl)D~Nk5tyvrr{33F5IYWXt{_I;=&<t;lXI#?mSZllV%R-t<goW%-575d|FqS
z%f(%1+EUEXT|nGnaB$_J3qj732}q`sYYA}$4;sSWJT?&kFFGZJ?-1e6KipT3Q_8@s
zG_=L_8DNuPx$9d3T(orR=O7SY3v;bTI5F8?c%M3;OTy!rlC4Y^;uQfCS6WM^+^A&y
za5}h4IGr+$#*G4;2f|dm>uRqd%+;^Wu#WV^9w2X*(j{SO!?eEsp^Ea96F(P0@(6~R
z=xN~v8UQXXUfXqx&9bzVKo>%H4b9k70Nob1tZ|9H%_X#1V~*Y;EM6iwN^dS87Z``8
zN%U}~Fc+B3Ij5y+yeEiGgI7kz(;Cep06S-1+G#{pka^ZUq<v<L3ZWG8lXF%d-+A~m
z6UD=#DOHwAKQ59=^S~=kANwJk(qPgwj*Eo2)-5hY<$SFgYY*f&^=dPwh%gIK0pV%v
z0|lh1umcsL{_<~@LzbmO1mtF~H;~D~$~aG8?jPuiqz-;r8B<C)_)jkb<yo~E3$=(y
z@#zzTvBY4JN_rArEws@Pn2U!%4L{m70eNS`o<#GH&R0ExUG*Wf*l5bbgt#dX*kWae
z_cuZ0hV?rh$1;Hh7klgBGCHagED&prfSB)Wt&#PJb_R%Cw6v};1B8~49uDM?NE&T`
zd&bN3?Z|9*`r_FfwANA!#HyqGO{LPLAz{OQ0`itYPNd~+1TEDuP-n@}4&hd8C4@Yt
z@G9ir%74*NmlWX#KjUBLN=K^2UZQD98UGB#x<7{^Qx-!XtTz*o8d@sv4N*i!sc?uV
zhvJJ&qf;<a>FSEFM7xQT=1)2c9pV(^A+-@p917*0k+$;53EV0I-3n4<m$~GiATes7
zJ08`yTT>Ray2TM|zZUGwRQSdadyeZ51EaX?5V>@laRijp)j~VKzP<7bAXi_uHF2|O
z+6S{Vshd72(rI=en&vdqz}5iFMMi+v>eeYPZaoT+jPSXS6lU(Dq1*wHx3Hwo5a3qf
zs4p|M<1&Iq4Qsp@bFmOMupJ}uh(76aY>682L?*gq`?Ah4{G$p<;VaG%<)SbSS$gFR
z_;G<$K^=uH*sCH*i^X~%Ce4H}2RO>7z3e7};6R=~<5bbEa<Z>e(dIoCj9lRfGyR&P
z%rm5F^;r5iXqO`CtFVvZx?7-G(4QRa{X&3Q!1$9=-b^8+iq;VdQzwXWzSIq=4EgpR
z59WEB{h<5?Sxt#Iu}<J0Dp^Cjh<fcH(ur#ag2`7^h-=EJCAl6uNRW$!+6iOjp8|5@
zvmK2kPlP5>x^Hh+Z)_%VyhGuZMq8Bs2y;eBt-Kmnom52X>BEVrNC#k3pnmy$iO8~r
zjoaHoT#&AGmoV?F0vO|-w<{JF;H;%{lH9aU6*j(gDAgGb6Fh!-&tX;aOpJg=<H0gK
zB|*H-4Gib%1e-C;ZxX}A6J>7loQ@9^%qWG?ZK!rnEPOHqqfbg|zty*k#;(&(fL|kw
zzzv;#pAg~NHL#iE^^;!$xPRD)j$LYOEYQ3jM1yO4k!}jy$PdI%%Q)0X8w`BX+)@T&
zbsmZjOPa~772iIEM9lkPHjRv(g?<H+d8Vr+?PRFjse)Yxz3Y>18BP;szKkTL;?YP@
zZUuxe2m<=zbOm$mS~|$bon<_^Jd_{LmjqE6JtL@Y{F76m&45;q-agh@_E>?=PnUH%
z4Xkn+5Cej<5u1;LPKR>t)V3}&>8PFoW;U@as>GmZCxR{+YZy0@We|LIU>YUbOc{_4
zM>ReZLTgS$E%!TT@~xRhR=Asr;Ezi0P#l;4!0Qx>a3Uxe1*QAmB+9!paajvPx;b3+
ztQ;!IJes0hRKy6a_`PVCgL8s>q95>U5Opz>9C_!%1eXU~5{htGyvvFi)pJ0c8riXQ
zi*_^gB)>ol{|3N?;!uwl7e=e@W<t1=(x#SCzqb(cjME?DqtO@><ZhzfCN@ZE`gu~Y
z+YL3DbXC@hp>qM<Ec9k3cR2b>VGbzw1oFF%FDZe}XUYUSvwKVsot{99ym-2d#!ABS
zEm7I00^#*^-s5e?M&}iY$VZEgP4CSHu_=}tuC>o6l)0mq2V$ydm$B8?>2wb-F$R<L
znOpI!FqfOx3~KlqL0PeKL6I2!v=WD22<>_#nF7c5s|9)M!7_w(ae7Df<}zUSP&?CO
zHEtC|6SuT*u62<U<~);*dpnDE)1(`Lfp{T@K=Cw9>tnwq1iWCjLv(SND9u91$hN(>
zh&$9oB^4!kA!IdzpQ%{wlHzZ<5Mb^&K?r~IPBFI;B0Ohy<95AOo57x1@`yDqoh=@R
zr)FFw#Pig&DJ#dbB6I5_AwwSw%l6B_%%Hv&Cw5l9Z(af9?kP8It=LF_;}MBTmlFS2
znCs8(kSl9m=P07FJ)^JQSORoEqzreLfS$eNp$q}k3)xOzOuiDr?ZWvOiEV0^p9!UR
z=+QM2iMK?%yy&x3YrV~w=PDrAvW|hrS|Z_eXJtnGsQjy!xowfF`HTE`47w82MYx*m
zs+rVI5<tWWoR_cHcix{SMbEU$S>+ng@u8LuipY*o;u1_5*+?b{>x#dFHg86BH0X^@
zg_whA$28KrGP_>uSS;8{KDe7`hv5#8AqJNZ5~fL&tKNY)T$Hm<p4_ue;4HzptV0bh
z3etr3!{38a2)hnfVqHQ!(0FmRH{KTQ0mN0Zyn1u}{CNo?y=MX7%ocN9M@Wjnia>Su
z5j8&W7@!ojJ7$z1?%vqeXzlo`K=|FoO=~Q9J%qDQ7Z4YSrr9JmXG3*^NVq=<$7xsJ
zz_*)*-4>GiJ^lcYd84aA_6c#1Fc(8RH*1mX`^O??+Vw|8fQrs(6|Hqq6sTAIbEQ8)
zC`a1+R(+Gf2+x1;UN!n~U2(>ZV9o%q2E2ZyLr((TwzU};AGP9Q1#_z;Q30D-iP^6f
zOda&dq~0@)KMQdExf^ba1^*09_HAvGVnpAraA)5ru}>0E^uKfw!Fuq`n?TKm5gdZE
zqvcvan6pnCSSei7*n0_firnwA$-k)t@Z-s-{_bB0k*mk#EaQe|oFx!;;kghFy(h{=
z<`6O#ABlG5F;|hY%bYiZ<ZO~Gcsl_(RrEOV$jqQ^33Ry;veApU{TvbIHE((!YF*@)
z1eq5JC(m@FffVmP(ar%^XO#N`0koms!HKE2I5dNZgh!*dq%r2|i@%8W+|Xa?ij{7K
zbOjLcs*CO@L|s%p;sX&blXPLZ%dXb-blk$Xfx8TvR}pXtFw@a;N<&VGZFoDdC#Ur2
zIZ<~L<Q(gAbA(;Z9Ux`~vMaW+tYy8zI6kegLY$({%uy~>@bxBuCaDd}+rovS##@bN
zL(*9PKY`9fvUCJqua~J{_Q42xuLLS1m<h&VcM``{&<!Q`R!Nxms$g>KW=a^1>+gbc
zh2+Ujin;%o4Pvh?q1AWpCX_oX3bp8Lq?DEr>UwgA#fcjR&k&Z=t9K<ur%MGf3?{Zs
zj^pnsqVS@Hv|PL5@ArXe$V!g4>HQEeCmHH#I4=2s0oq^8#V_cC4xaq*zbHcYMW~QG
z#HUNcbW=^2hqnkb+q%%!;R645A#@i%uNMnF3`yQ7TJ*$$B0Ly4PV+jsXvgM8SbNil
zg}CY1Kp>Ml`w;*oN0T50S^QB*YA>T5F5@)oJt)Y{so_kApocH?n8Q&iuK9G*jay1c
znX4bYFNnhNKvgA{db~&nBNy(;by_-ApiA5?8R;@lKvQDw<1w<PXU7>razD;?)Dort
zR<KK@`+wcNRAP}Q3y{?7)6o%T5q-)@QdAC9BQW7DA&$%1F<eV7V^@3{5Js@u;vz1d
z6Y0j#i48k}(fGzQ1%`xrkOKF_k<Ws;0y?h6=c4JHve)j46`q4~XOf^*!p1EH<+0V*
zWUs}_bc!%dD4cH8p{@{}TPbZZUid431{i0ofmq;q|BX7m-9P<Z0DSK0<H@H<FBFkw
zDuEY$0WL(Pfkr9@1xvpO<XNNx*<cHso*}$B#y-Mb8_ZM&<1a5k)AJiBSZwk#-!RA`
zf|x1NL786~@n6yIYgG&w(26<t6%gmE>ps1*vd-U}2Ii_f)ECuPix6G7R@KeQs30ek
zwt9VWq7d?B&mh&>dqh%KmJDtpMVL{T7Yy<G+-m^N6HmR^iZtU3K}1m9WAWD^s8AP9
zEtY?SZ)c@TXlAZmg=l7BJ{pWGL^*MG?kGsC{U(4X5e^_ckqqNnai~xygS0^siC+s*
zj&-=d=UXP&zE$FA5oTI#jNN7Z@6awTGaMQ~gYk1g&O}#yjXK6!I07zF2^;cCpHOGr
z)Js5>SmJGBrxkl*to07x{uh&Q9ho*0;u^`23jY;l5M-dbetz}cB75{glHx8Kh!fuf
z^@zbIpA?B0-4_XTYSSw{^!>{O<l4z5aIX(cw{U8w$>m%BfX)-#KxHJ&ohu2`{MQu?
zeZn@aw6Xe!MOZzrBiqQR3o~mNqZs+8Jy1e3|NGiWG@kuu5qE$|lXmKecZB7R(TC>=
zdeWqifHbX7m=s&<o0iwHpb<4ulq5Zf)X2dmO`H--eq5wNrBLk~i0cJ;C=Cu#AO-{0
z3s^9J0`9_U+|#_UErx}XXBFkEL-d^oA?Jjoc&E|7<)`43g&SHbeUB(tPp5h&<nGU$
zM%$W`;@<pS$Nqhn&%qfetgi_2qMk`lkMWpD&!c)#4S7Ub0$2M2*logdq|z<K)?Ao*
z)j*`7*98;g4j|`QIQ@pFt1FyAoK)h=UiQly33MlDn}!kX=$Fv&GI=n3MFiDH2_-(!
zw{w=eF1(nb=JAySck6cO`hP*V#PAONEZtG@^3B)4W=KaZjqrtpkj0ShLKYL@LW$!v
zJ}g~AAgo~xklIF7q$`N9Bbjl1^WQ*}j9dLUM&HhXi#u&q?<^5^`|Mxh6Cvcu;(+ei
ze;}MEr>#CDMI2LJ6=+5b>tWla{|n7<LH@|vNY6P+fMX%n?&wK2c&7+-ECzmmJe^}v
za!frignzA!rQ@Acm+>Gf(4CEii5la6qdVRgY6c<(PXe2?g)EaCO#U5~{U71n68NXV
z#~ypHrZD)?i}3I!ebXV+i+S<5DD$$H-nh><wi%xVbgjoxa}EHPrGp((Mb9bHr{)B8
zz4}Mt-zlP9c@D|i%EGco1-c+`NlP=%d&-FH2)g5`GU8B^RUWx(HGVV~5#6AbAx08c
zD|JB>Q%65e&ew>f4jqj?mWj_qQM+y$Z_&B=9xv~}AiEHz_4t)gmlvsWrkw*0RMTv_
z`8-9C_CDJ4e$h;2xcmAr|0S7%<TOA1CV%DHaw*8N#OimQKqm>KbPA8wyg(EhVWiwN
zo-C3O<1Yy^LOv0cCyi{sz$UX5Z+{EieRfc5D;D_kIWj#5Z4fEB#d3nEb_b0*S>NMb
zCOzn>&FU#RM77=%r_Nu5z{eRDKy)_Dw1kc~oSaN(d|fWkRY6Ho@07-qg4~wUJx(S@
ze-)OQDns6n%e0ch0h2eT0}B*^xR&Q-Ytf9fE(yOs7U3?9jsW6T3H+-O-v;(lgcO3a
z#L5DQgqO3vm?DB|53}M9>KhKDealTi{H~1HsC03~_HYU4J-^Xdb-^NO^cv9h*j|7+
zU2CS3%!5K)Ei6EKVzGswU3E@2eQ3fp<L4#D0{~{nvBI2E1G5QiBQF%<vDL{@@mC8M
z39*xHjOw6wt1xF)uVL)}sAyN6D$$3Q{!S5Td}kcX2qPz%Kj`r`_9#NgNs9&A3Xh3|
zUv2C1u+y_5;8!R5{Y!n1Z#TDxq?)@!pzn>+4Pmm*)8SzDf3Tb9g~(=o%nMD5<8tCV
zk->hn5^+Tsl=x98b4q6)GQx+pK=)D^Q_zt$nOrV3*STN2@IMNm#FM5@iTm|!?n@CR
zIlNk65is}9Dw{0Lf<F{QjkWWQturA(Pb|DB|F~HYHqyUh2O%UsapL57N8iNbKDZw9
zMG;i}jzN8jkX$q#3v9*d0?aGKLI~>gd_Zb~?Y_@HxlnXRo@<H61m(F1NgY<$V)suM
z1I=yFKuwf+pC_0Ex|DjVoRje7i-Wl0yba^1*9fDuD3Mp<pCVi~rg>WH8%qMX1nLKv
z{=|WToGhn0iW;+oxKH*VCp=FiDY9Bs<Bc+OCs%&R+&?UlK{UuH{k%&NA?J!3<UnjH
zz*%%QkR)N?k&)bCXyN08xGvV30K2D5sD~|KBYwVgk#c%TF=h%d`;wS58C1r~%M_??
z5A57HhJ`u@)bm*!$zkbbi;%5)1B-Rt`5QI+y^L!7?5m)zRT4r*mjD(RPH-J#@j|BB
znbt>RvE}%W{xcaTVJqv~5%u~z4-p7(p_4ma9+#teC+8?*ECVO#+YFOLiOzez5Upl?
zQjL2~6itGtxi7xGJOp{QwM~ij^-a^|7oUz1$pY0g5=XDVrxK9-DCS-f%0=K6pE@hA
zCe?X`x_6-Jj72?<*9&q<+HsM^ytastaYI@^ZDJL<<@E={Tl&*5J!WK@<%o_o5Lbz&
zoDKX~t@8uEDUFUG)i_2JrRG{oB8xRwf^<VAok8`fgN1qSXfjOV3DIzX`DS{2V`T_)
zm@O?zTw?-U>MBk+SYtlRLDH-$@!tlk6hU+mNV9w_#Hr|^6;!cTErM{bC85%aC6eTt
zyjmc{jMRYgRaY#!S`m|zCGX8epAeV7%2VTUqG*q?%1~!qCX(vtKoLu<ZUQ@>dTcMk
zeUWENeP<TCmLPheOv4&eCkS+<Y0rK-&D%oUo=6>7xiN|OVHt=w0I2qFgt*!$oZ}(w
z&jMUj?%x}6muT7wEm<TN`>$b&Rvqb$iYVv6SyDPj@vSw1oE{eqtl$Zt-I*<7G5yK+
zM9vA@>Dx84GQ9Qz=LmEq<ob>BJt@S!K@LH1HpCmYYk`^TI+r78P#;OCb7qswvb^@%
zMO1E%(duHlT1S}o6U@ytncPIT;%9<gk)(qH_bwHb`Dwp;yR1_L(%}Xl@$pMRRA>sX
z;9RC}GmZ(GM{e_f99m+R2}NSaF?86O&<5f<sAV|IONZ-8q2BGud3ZbagXxy%l!@RP
zA~1=A^>rd$RTRrPz}{a1dOf8Q+fF8g>zTT$T;EXMMWz7L(R7m@2Z(T`dAg=0-)2_`
za-$64+DtCR@VZ=v!%AAFCWc&qu2CNSifa`n=U1&s(LG(t-%>_pV=UF8xV40ETY)FD
zTHG!OE_S1}_2g8(%|#rxjl|zYtIM>t#a`3+CRgj%atyjrfF?p%8c&Gu%!C>ZyAcfo
zDYHymmp@=$XN^S!HQ!9qg<CvveY8mWHycu}B#sf4vu9R}cT2h=50MM5$N#Rl#`|Dw
zDj@e*+L0i43USMFsi#YGxq*L3s1s)8oDm1EUnC^;ou23xKwUcfMjG+T2F1Tz=F?X5
zZdd?pi`ZsN_)!7KzOp9{7LZph$*Fb5j^hQobSyKPT}~9@M#m47R0rd-G7xgHE?Bw9
zMnzib4kl^xek70rCa2kRZfv4kqQEOfI8D^SYb^5dl%PCBSy$PWaZ4Q^2&FidWo!Ge
z#3mpvH5)4Wu9Pt^-_*fqp=37By={pgSkWr+rI0+@U`Wil8T9ygO1fy86V})qRO6FZ
zEjHMqNEM4uJ!`S65Sm^~IIa#vIYpM0YIKXvb2RhdRU%zdy<LZ>hH5?`aJ==C0RMn3
zji3RIuH7)ls(vAksJS|MZjOTlI`O{hSez!3ZpCG~*W!!I|1s{2w?%puOc#qXJX&Nc
zV=_WdTF-Fq1Yxuh?i|MA+O3O7ohW}M=LmNR%T2{bT6>Fk1-ah59nup=Yy+(U!GQ_U
z*tSsSYu!G{wHr#Pli^kbL%Di%DWGQnRN-b|A(S)#`UIOp>;mdc0J9-Hq@`=f%M^xI
zLdh_uZpSw@Kmt#*i{Ot*D6lWReUT8R4d}$hib9;DE-7k*a^#Qa55ca_NO!#}-WE-Y
zy}WxJ?}{+rI~WZw?co17q159e5v~-*;Oty^A8AJ*^BvtQ+6K|s(!!jr{J?XxNtZ^e
z3C`K-renJ@YF6Q9-z#jK`*b2k=s_ja&RkyMdhu34l(Ef9E{p60=@v-4c;(iLlZ56L
z=pL$Ssk})L73I1WYw%qlJW(YFU2%K~Fex&FEWT@*uBB~i*}aA49?i8P$o@h!15cb7
z6@9xe^ZW^BIXW%P6qp!HPyM?Hb4>2L25QKD-pc`)=x~C#o0H>J@wivtc{b(xq%Xd?
zI{>-Ml>x^#`o`u$ay<ua3%$f1K=6m@U4ox(k-3KrC1(U%?+M}-l5z+>EK*WC3-;p5
z8GT4r&B^fh*TUT=nhcxUM7jp8p?2O<V$HZ;nCsn%qz$JU^uG`G0%V}j?YN*?e;=rE
z#vwDWM!OLH@4`(>9m^LeHzP7iz3w4_{3k-4Fl!SYt*h{B<9!QoB)Kop)+m84R;2^c
zhzw#jRY2E>odLEXdUAwImB)qCA`(u<%OW(oDkE6X_<It%%is7h-{$N99<Mn-#m9oE
zLptP-NPd05PmC}XBeJT33<_~Mh8ZP@x^eouADG**#q&eF94BC0i4t`~)PsEnUUz@~
z$Q?+|pOR<Y^#q!A_}*?uF0!8xH_1wzYZ@Y5RIUc_dWb_GL3vbEB`H8}3KM9dW{3_)
zGJZ>JUgQ99crXPoKUICZ0%`xJUVoY}58g37CUU@m&^dMKp&-#=%$+F|wql5a*y^Vc
z6cL9iTF5#@@*heGmH3~CoJBWc5eXQh;s8ioCd8H^oc=J+!sWSgJE7(Sesi>6+ewHS
z+mFHxdfPdbVRpxHLMFQOM%9%f-09QKw$6*bbg&l%kF$<Rl1Xiw_9A3DHRDJ`YvuNW
zsR8bvC&Z6MxPS=ddYL;Fvs0*xgOHSs!kGeGV)j<Dq&>Hzh=_J0rbtQju~rS#3_(H6
zkGHQa%!#qY!NSW+KodE776RN^=t9hmYWNL00iC4|$GDuntgA>!jn3P@bpZDQ%r3Dz
z>*_`!408hE)DXEf{=!8#9ds0VsVgoJ;!c3vfVD!jv-jiM;?jxJ0U6ZngS&g9s|U*5
zr7zJ9Tk(H_97_*eVCO>Bo)Jj@>qdrjWP@+_ZHxfo6p?v3;h~Zad#H@t1-k)cQ!fKQ
zT64Z$(5%4C@r7G+23E4YSVjLhyo@kAbd^<&RfW2e-F>)gsKr`B%=JNT6)RB_;3}l!
zHnFyE0=Y=(BA0{c7@6-Z*jY%f<(KDU524Nil}m{ycI*T3;Kx**=BD%vGW3c2`2*gi
z{d=q-%6#ZW>wSHZPQB7E{VUEDwOHvOP&Z|J-8=eput>Ob?H{VOGtDK0#+6&T=vOqg
zG`EIvnh1A3mL%I<ydZ46E{H&IwnF1BI#A@6PEAr=()4hWaFRnJiWQ;=bB9$|FI2Hk
z@etl%k)lN)DdrD`KwVA>DPNzHgt&h&5KsW1+r%LPT^wvoSas7mcbO4TdQIDU6XPs>
z=a~RGLoJ>Y;NFUNgTa_<1~lx<ujI}x(&5qmWIwWs0P3LUlj3?2q+I6>zIXLatLSE_
z7E?x{Twv}+FpQ4Q65;VTIU{*qh#4x8p<MLEx?^DOMv|6D5o-rQx$Zc)=P-Aw5X#IY
zQPL&c>F3b@e@D{|`+I`jfl&ELy7pc@RyvqJ@`R9Ncxs#k=0t|t-SZiT>y%+rzf2R@
zT8EH;i^Rg9mHqc6!ebv?{>39g%sF&=xM7_CPylt?v^dXo7!-WQag^@d#zex_Vd)9a
zeYgphuz^@XM4lUl&{$#4Sw)bW8Q};6BR%GoP_wgD(ZjT~?&=EZ!NFx>Y*VHsg<GAa
zzWIwH#7f*$A0(1;RP<ip=^|VOW^YcPAC|z7wlA^Kk&dm~QpgKLnz8lt^q5A&>PG>(
z0zE^B4XN!91(|<ccFBCAknz62iCgRB`NCXH3Gp%Rl3RvXgsOyy*WyP<L%4)=7f!rG
z0!W!fBUvI3#<dw?>mf4-M#=w#xIiOZGYp_@vC%O=x!|n!(!%6b?Gj`Uko8xd6G)ez
zIAOiGVkX}%Ae!wOajNfH$AY>e$$1jqUXm*91%e%zX)kRx2z<9tmj%THge%q9=ePpG
zI0!c(197Gx7mw{7My?2sVx!{$H4<^{8Rwi(MC#{~Smn4v5dZUX4lc$;Wd0Iep08q;
z6TvikTZ~S9n~{2cq+-AY!+;NYHm<l7W_*M&pfm@Pb({q383cFPjo4T;$@X;P8%y7A
zX!@I~a-R@-WWJJlPNd6)RL&Mx>&*gF3pX()*-eD2&McA+s5qMu=wy-ENGaogLKs|j
zCi~r!p<Ej&QH;jP0!Xqx%HHDM@?REYX7<+vxM_yQ+G{*gClU1qzXWyB@CSs^Fg(3e
zm`j5sjP+kj^}hw?($K%S^?mh}BAB*yEUj!O{vi-C`p^(YCVZM@xZSA6-6hTYa+6}!
z(~9^E@Imj!Hx}sf^w%3S3aUk02y@9>%xx`;Fb@>!wqsLBZ>YqDf?xo9mS(&zf&ya%
z*TJ<OLt@v{L1-1+gz7A`>=}?W;dIV1H|m?1$(r@%{6`zteNW~;ctl||hR!6EYt$nN
z@hs7)J*1U$iYOOd_u5I4wdh#|mZYze$Zjocys>%uJ+0B(lmU?rbD*bf_7-NA>xD^>
zkrI*Te{5(D72-~SN*7zCEq@JQwlH_Gp+ZnB$i+tMWLoTSc0nUl7>dV4=k7_tQxE<~
znA1hOokvTLJO{w(O1}jgj+peDA|3R}hT>PEVIo>{eQ~LX+>i(}al*wLJ?DaXY_Vra
z{slF*7CsLcHcE%J65EM#X)!EqkCy0^Dw>Bs7vW+t72@jm4FS%E(VkvbiEo_`M#FL*
zjOFy51)VtX#q>3;j;**&f4Eceq$`*AM8^%u1qI5A!w^aP#o5qtYc~2%LfwwcP8vJ#
z<R?nN<>7i;K4Ax9e_>`FCmyD)O59pPND*c86AuV-YwLJ`lg1q{1aW222t^8URSC!l
zxE0%9L<oCfkS(V~nXPWiJZ9=Iv$2;~#^ZR=PTqe0;`}m1tHC|*N*5ENU=>ce>RAn{
zQ-640nRa@~PF%;h5@$n?u8>wM%pVi(tXj=cNn55Xq!F|!q?B`5nVFvAODssDp(Dzm
zQU$>l`A8vjk6zUEk)Mk4V&BfdeO9Ea$-2~xqb`A_w^U{z5<5qP|HK3s|AeC5ICyo(
zl{Q0a>ZM?$hmNrPf^BwL5rmzf?r70c6y}B=-@<VhEYRJuKkg9WLLJ1^GwpInH=Lis
zI$1!jKfMA^Fi!|^_e;Ffai|tg3nfv`z3awXWu#6`aIM(t3jWTOl@{B9CCx=}R}6@D
zzO25?`Hu)7JzA$0Yg`H8La46Tg={B;j@gfO4nZ`->?FuZRPifUiRG^<Vqr+dPBhLE
zLVIu{#^uQ!qTIRr<$RLe{as}gy*SGVKj3Ntnem9SaL~z4@lauT8sr56bY-wANT5?g
z9yuV%P=Z`+wsZ1Pwc0f<K++Y7DWVwd>ra^y_vzbdvIUc)Ws37=4xTPrV%qO=<TTay
zh|1NaI-}U=IJGWB-V6VpZx;qpqsDUVDTo$e6c5XU{4UWh99n*C592u@PEjjbB|e%h
z71V?z|Cs+ehoS#rFb3CG7T{#0IL(MTPl)qIKQiqUo7V%ma_LQDS<J=g4S+O;F944h
zl`~KFu%8KV!ss+`ERK#p6bRCkZZ;nm332D;CD_>N{~(<WegrZ0*Op)}fw_Pfny{Ii
z>yIGg1CM)+B}6+!Z`#qtR3Yx-GzRy093iT4zff0<A(=D_;zl5PDJPw3?D%I0r`tY+
z6NbG-J3O1QWKK6G$hE|bh1=(cOMtBoqL%q?a*8}9hR@tgXx9=aig0o~FVDScS6nT~
z$+6O*RP{Rnxd?1PV!ppXyZ1`;m%Jw4a&v(VR%Y~9Wb|`uiLnHZa<MrO-?)Wv)J@8_
zvAzf=%nJaO*f7&9Dx3y-V@Px{OgIxWOpX#p_nwLra;Cn;DD|1IZ-sENsRzsWg0}&<
z1!RPZ_;_I<gx4KIye}d*7*1(<<per+Cn-%+_0gCh&?T@hS*oy&Ftd!yvpT*h;%Pzj
zy6%deu-fCUB2v;BkQbHbgt;)Xfna2Pni0;xOg7T|_)JJ{+J+u<p?BO3G``f}xt!aQ
zBpP@_I5mOnj=k=IaLLg#CV<A&X7>WpizV7+pza_NwlzB&@hg3+4LUmTGAF`pL+PE#
z`egy;OHZY{8SC8#?UJL+E1~|@C8FIhOaFZL6UHTIH8{4VsdL>D#NI4jdgCQQpc%?5
zOA<^~<Gd2%n8Q8;Cf`{`ZVYt}jxe|J601Nnh_hKwjV;t%4-}>2l6V+%g@uJU5dNmx
zC1PGdkopWa?OW;F1&}M>Budyb|KHX&IXcSdgXlABKvc^pvVLO$Z^ei(vrUhiq6JXK
z8$iKY@t!FmSc36%E2chJ6i#OZ^@BAY0;2CA{i5DmipqmejEtKFxH_`$*E3`<J`CiE
zL`t&=aJBd?_y{PKoH$|K*i7HFFpEng4#<Cu(NI-==i2KzcNOziVQz3*sW%>Y6xum*
zE07!zJp33CWvo^7w)EqoNS!Ob-nis(hn5o$`lsAmZuErz$Buq5Iz%}2Ug?z`E;^4&
z#1$~$;}StxS`TwB`Xphf%A^UC;s^S6Wi^T=&!o$=3C-;1a6B3l1vpvsoV0g1NQkS!
z18;2ndG}5b1xKZ_8ZU{UP7D}sSl$sumOQu`-+G#FGfDnLxK+Wwj4-0I6Im}-dIrK}
z;>v>enV7a)v4Kzz0?t<qYh93TA~2T%S8`Yw#118--LqEk(*)&a;c|wvBif8*a+1}i
zyWM!DjEjJtcLwmj@hox8;7S8$bn|Cg?xtF3D|LxBtI$@0Rowz|^`!JJE2pD{QT@gc
z@4My%key(KG!*U672)B3Z9oFcZeioyRNiN95<n?<5i)hEFK3H{g0V!+vB&d-@$MX{
zM+c3qN3@eg<^KAM=;QulW{Ec2I#Cpt?eiIeoF&XaAlCmv5e5|{35w!!K~!UUUoF1S
zxB0?GtQJSU2%YD5Hg1T)m~tNyNIp8i>opm+bBn$NL`%axo*NM1Zk4w4)wo^=S&Y<h
zkF0MesfQyfSltao^JPGHj$U@%NiPy_#6q1A;f6T!99bf~N7l#S{91^QgPtyYIlRy-
zz^*uZF2ZdkAor=JHZj%M@^1x{e3k)?zFI&$7BQl|#&LpJ6Lhd(v-Yq^mx(bT8}bi?
zjQ6GXdbGV(q|w{a%UfT~m?;SM$OJGh5@80kqkf$hvUpLTCN$lH$H!$5J?oj|J+boZ
z1fd`v&vBAS(!|YFEvCK!F+Sk45FMF^P&0u|8E2P0N`OAh+Yti-$QF?$YC*S(a@}ZY
z?YW;VBeeI8QnJ722si;69*srcB!WkUrZW5yF?io9GQ}32+ljpexNPasPZk$a{;kkF
zpb!vsOGbIAFsj6g+81|;aD(AJk0Wm-?#chlaX_5(cmAj7{J|mR_o9fZ=Mv*X5su1T
zR2o_z334|X_tvKy<;L4ZNUr~qf#ys>Fd<#ZBB&gU_k}pF=5{2;sn0L_4zO#0)&ok9
z2u_w4=B<auoBZdcUCPD6T_V3ug^)`SjgvjfRAbV+01T2bESGBf&f`!vubraZXE-sX
ziE=b<FG1!+cA}Dl+);usHLc}caZd@RSvI{|I~%A|KN3}TKrg*VMs%`t4KO3tejn21
zX8fVCH5fAmjW-+uQmqjSe*i{Xq?^CENK|gQ9(Am0EdF5u)tY0Ya)vlnkUJauR$iEo
zO@+XKiIdu5roP>E^@=18um$94!|HCd%yR!MLSd)PDu25GXRUV~^px9u9|4(_q%<0v
zd<^aKr_+qef4wj_8yW~K5reP^X8uRGlbcR+vLeP@p8&ZWoV<AQwSutXn*!;5>FKtM
zMK}XIPx~7255nA8@mlb<C?~6DXtgz(`%@ruL)kNIY9+$4Tw93vj4<Yj6yLDG`Md!7
znFSir5I{DlRAON?-xpBMhDQSA{wr1#<gzhU^OQLohn0jOZEf$0RrTo{<iL9%CT9Tm
z1(LLJ^btV+=#maEdw7v#SWU0_z9q`6Zbw5VKKRmsxipq3$XZ`P!X+%ho3VokVzQ($
zJMSyXTt%opiUOMi83J8cHo}Z^qWx8v`9y2V>F);rDi9s4Xz`7|2JwWW1H9i#HEM3K
z^~2Dq5?d>vGe~<>b%hziDA#~(aYQ*g^-;RqLqc3q66gIV$@{<k?;;qQC&?~Z$aoXi
zP!1l9-37s&$=R3Ap=A(h-^lXrAwlCa4d%K?O_ls3f?+zFiy;h)-V^C!r@f{`PX7?5
zhTv|e8EgNi2qAVRosSv9+&De#zv8t2LT64U%a6E5i2251(Q4r!gfn{Wf1tS+agfzm
z>?$Goeq-v1GX>=al-Do{){1`#%-ty&WPM!%%vk8sFEhus<I(V$=I1`*YJsj_)?365
zLfCjIYMhEk<10aMh2z@*Q|7PdERy1ZTh>vvW@Row^|FRkfQLjmVQww8FpS2L!kjQR
zf@xyBRgl>|9d>U!Hzc)S1WCD~Bc{w#BqP7AtQ({8nlSpd{G7%j--Ix0laIA#EGvX^
zNUDv)f_b6L7f$S4c=XH$aKOPJL4euG!f4CYxJ;nir-uGq7t&_B)t`jA&!DL}*iC;|
z&EFTyWZE@^EcIJ_k2ke+5M$2yLGpx`rZGebnhTB)PBjou)Z-`-Br&ONYTT&r+<W9%
zb>9V`%{H7+qxgb_lOS`qmnAug!k-uBXoKVs?caum52z1vxp}@QmkY74&cqj$2p4)<
zIsQ*bZg{q1I^RE-6OmxOKk5q-z|7<xlTFSo0&*)OeofNET^9m#IjSf+#2zJ0QM%$W
z(Pkp6AS*0$4b$KJ3j@1=1GVWKhoUAVR|tVgoFLkn>G3z@M$NceSRn(KvJxwp+T_zR
zF#3*2#pr(tbEE276eos7$UMuiu-S>hJl`Q*vw{?p1Zk()Au^(K{Y}t#GvK~NP4kv8
z*e*FiA20hr<?hE-hTiWffD!aJxFQme>&;an%ifp}7gHu%c{BXZBA_m2Z3cn?R0wOa
z)S`gqWjE{8_@OSrE|*@O(7G5aD5&$n_@arUlg)%UA7p*<dan_;cNq@#51eVmbRn(-
z&XTF()F=QwmC}r>1<;k7*z3k~BAgK`5pE^pw(o;@1Qgm^5;i<9Ja?x)J;?U50P&8E
zc{3he42o2-CByqhCUoZ=>n&a+NU5_gP*$TYgqug|3cBOWpqWAc98FJrq=4hyrkBS)
zlL{Z3`}BtcBVwx$j?ps{Hr{RIKPGjsKPoRq>?msX-AnN8D&rzh`pZ2+T(u4w3VBQk
z^dq|OiGK;u4h%8)BL9!8^MI19sQy2Kh>}&VNS3gYac6dC6G#>j5CK7gfb>lF<mn!!
zduFzm97GmD2_i{EB`8T`0Ragr0)nE5NLZ4JWCT$V1%coH^Q~{qdwc$K&Ybh^x9iob
zx^*kvy7$(aMQTPlUf*AUi=k;%dhvKym|p1Yi1)3<GhG8=O$wqlMZ-A78R1$bQ7{f|
zc0FdU4Uwg(d(g(z&Dc|@KcH)6#?S!A4Z>Ut7E|gP>#pOB5UfhvB!VuX1uT6`wla_8
z-&xxX$MYimx`KEjC$9)G6Ie2`c3W{>0Qcx#`RA>_p0i}K<#0RQZ?U}~=fV-C%pMy;
z+*Y;udB2qw`YU{*_`&lMq3zRB@;my$hGYmf(*R7xXM+L!GX0Xv*v5RW9=})uA<8<3
zEbFXaq+$D{x4ybTf#?f#rf&#=!+jNEl$<NV98TiR5djO&2Zb`TO(wU^`0$2)L&xrM
zs0j0c7(=zFigs(rWsNw|t+;a|u&GhMr#67zHbE{u@j*DMiMy4EcAv2YQ3K;`QCX@v
z(Z&*ug|t|B^|T^?20d9+qVN@lI5qIscv{b7ESp0@4Wq?g`jbXznPl+AIf)RLd4ez$
zF?|zgmruHKc~k#aQ08u9fv$5{cGCip?GIz{G$C-VQsXN}ipT>)>DAJXzY4eDaBSW&
zFIL>lZ)q-$%|&?4z-?`@r^q~tq?zP=0mUrc$1y_MFJ3Cxg9Gt(xZz1#<9h_U78(O|
z{IuET1=Kx|qSC|}g4{j~A)a7rTNKH(?qkayhYNE3vFFtEIVl7t&zdtmp4PLACC6nn
z94WfndRqeL>4$!f%88EhErG5R{fvxW72tAOD4Vh4RzUt5PV)GB$Dj~+OV|&N3qA#b
zXeh`aR@b6Pj2Re_-_<$q6EZdWGfy-2(}@>OcRJcV-E@x;w-6VbQ^*EJ<FzUy)Sex{
zE;hCl1e%Z+^cMuW*l-E<?k&J&WJ9ZM)15-H7_p&?=!B-sDBLid65*EZZY+p1W|VIe
z{fAlL<`N;GgwBVa6vF#h4KlgT^!prCO>*uqD$-w17e3(+B_#6|lU5eRT>!2feHTxG
zo^gr)(SqIJcH?DKeNdR+W%!XD_v`}Dg1Q;iIRLcwjE))c<GDPWKU$P{?{ZBEqQk*h
zQQNEpX5ms$lx_~fZ&gB<9L^4LWOmv-5c7=@g?+7PniriQd2GHl)YKGJ1nb&UD7<Ou
z9;QP@kO8BfJeK0*Of&G}$WV<3a^eMw+q2O&ev<+cFjRyKgZ-VtSXG5k(IK?2ujm=J
zOgSUOW}o)E%td3dK!oMCO>A;j6!Em6sj?<LtG54}ZtH9$c&ri$PniLq;4_cU6Od)e
znvR`?<eca?^m2_t2bQSf7;}fIVh&B@?8qN&6Glpl1+W#L7>T#_2rpX<Q8;;S2W3uk
zVnf#{X;vDQI72k4wZ9mNvqZV|nlh}D!Qm6aTxo*ya4=MjZ|?v!C1;p@&}f=qe<j#&
zbMweRO=Cvv2$lx~BGcMPH-WAyLm($S-B@5bDGiKLhZ6?9S14APrIXnQn|9%rTRhF_
zCH(@-NG5W1)9>sA;@o9a#vvZpH^+fHgQ}wA(nfqol-W{6M;I50maxy55!-%-XLZ66
zmWw-OLZ<NXxn20h73>`+ke1B4b`$0bP7)N5!3a~Z1ZGYsmRpsr+QCX_K1v6o^?vKK
zKzVA%fJUPzoRERa75Zh0ZM~c;X)*q+Kv$KJTndVO$gV}6yozljL3#z5LBnlLjjxw!
zdeu4DX0G%((wMpI2a^9za<BQQaCju$Cx)$}Twn^TJHlf#L9h+kz?hmyC8GT?jS9^;
zNPu~ty#IVWF)q|2;qdr^I6|bkL#Dd(9^3lez;b`k?u>h{gqQ^>9$r#8{Z}Au!yblP
zf_I0QDj>Q|IC{GvejjdQjq|98+@o-{%~RcbLUX=~?ZaWfa2&M<AT_UZ{J~WgqMR|)
zTCzCZ<?}!m`6)wttwp{s*ePW?K?}svdjh#8y$r1g<Z(jGrNLxgkpL&b#;}j=>s3mU
zB|-71v`u0l;R`?2FIf@>nzA4xXiWmm&_OJ3{-ZS1nk%;!#AaWBaFJx>tLWATmXHR>
z7PtF6EXcg6Yo}C8s-9;`G}&4;Sz+U(u~(6!I-+EwV{d<vX_bwR2<L?znD&7hd48}D
zs4IznMvk=0d=Z3B#(8bb&@&7tzAeqRhbWgB{gG@r;}k);nYcyPE8ffZ1N54Z*9LTN
zq8G{QeyPZSB_BcW8*y+6^>8>R)1aCl3NQyB(_uYZ2wS?sL3+b&_6PH*Fg8wrna_x(
z3aAIQ&SWavSBOi+(gs#4M9F~yb1#^zAp!}YatzO{`7xezfg*kr!%<vQ=7I)F@&uQ!
z6G$UgWa99z10dYMeFQ~1_rT%{lSDzQ#_9(FP?J7Rkh}EE*BXTQ9Eh)pGTW=V$f9Y)
zHC2!eLI94$@Rv#DChOLPx^>)8N8)UuZU&uAv)Yb#1Ub9z@e~nc+k-(|TREcFVow3;
zlSDF!?;HX_!8%zrT%qSIy0+b>4ek->*O}j#Umh30>*H83#Zret@Hqre#=lIIANdLh
zoWLoXn#Zt6Dx2&UqbbV0vVz$nhVDsgagksOF-*_CQqL9{Mb4CE8cv3Exu59=*R0mV
zj*&Bf65;EL+tR3z#x+30l851+2{e~DQqgT0@vb0BWz(as9w<JKgHUXz=iD1PnAJ47
zNRWA-D%n%hB5iSrqG04`lVPB`B$_mU>Im)li_*G9(MHltwDKlZFtq}9i?Nvq^M%-~
zEmq>mL}-p+^Io^GV4NnvJV5i-kKy)M4MhB9E*Xzqd!bx^9kjt*hUzE#09_^6P1u7<
ze5@ZReH`gtON2SD^R|XhpP4?l)(?EXp3GpGJ3lKx!qPb_mLA|)QaqmlB=!>Jo`_a}
z7-1#Nbrg+wR5Tw<3fzwmI-7A8T6^f(Rb<yfAAUsu`Eo%B+x``Ypq#HFtH&k7(5|3X
z%${er9|3d)Ie<tWcc%z4qb6l$+8>LB_(Lpxne(@*15lZnctPH)XE%`6^}X__>1-4)
z;9k9;65|4#NEZdC4Zx+sTs}sMM(jQcZC?6dO?q^#FwfQxep%5D@zZN9iKUB<6_Yll
zbHS<L1eRFm=}lP2?R%r~Rwi`EP3(O*zfgmY&S`O)p3MWrTxQu2kCc$2SIM(w=P^<_
zcXVf5UxEJ>h}pub2fyvrn$C3Q?CJ3fJ)7IfoiW|(a($}+HQBOkMHlmoKnjVid?i*L
zcNU$~XUEBUcK(=X67y^238%zVWOnSFiH6*G-x6V7c8{@W-g*++UzeYRbgQ+vT_ANP
zs5O39Je#MZY_Z0et7DBeSZ;U)Ng0bHgitgLAX{<o0tiN@0lXm5`0srLq(9lhjw;R-
z?Y9T&Qdh-IM}i=Z=;m3x-Y<%;S0}KL(X-#3)avC8(Jl>4;RYUBeeI*bTm{rJS~vk7
zEYJz1+?R_-9Dg(zg+yUVG&>P|rE~hsxaU}&J+Tb+FNpg^k`AZ*N_^`$2<J{nR2F!;
zXzn6`yrB8E(i2A<4@q9A+2Vy0c&2tb`AlAn@xc>~Xu?Nr#PVemA+A+(ymugW679lr
zFe=yR{&?oAU>*>M=!9}oV;7iEOCWkq%)y)Syb_w7gsH6|T`yz=PUx0kM%R~@NK<Qz
zX-l2NTe;ndzkpd23aL*BbnaX@u02S6Qjm**nv(9u89*~eh5DP7?lBo0J}m^EVaU{o
zBfbXV`r$*`ihqlCzBsZn-H*oSP6mQyZN|pK^~}2iRo?xF2&WY%=EJV{Ed;W7<1Ii4
z@^uC9LS4Jw7e$zP3<G?hSb|%g0+y#h_EG7?fJ%1>E-FJyrcvW!p{|U=xbwj+zFuUh
zbFXyG#;7nCf<sWuv2GXON^;)?SFoK5O`$LW<R&Z;W-o5%9IT4xM+)Q(R!9TV#@;O2
zV%N>3X`7q|=|XGRO~>X;z<Yk9fQeLta!Q9mJuxnrQgbZ`3ogRT#)J^pj;$vv|5Upp
zg_=Da|50V?cT9<~qGeS_L!T?mC5Fj5GiC~ZP#`5tOU<76$TuO)1ICV8BSHYJIG3w1
z-q15?367^tz)Gh>`n!~w;hvFkJ7H#-_WW^>XkH)X{PDYb&itk1bpuXJ6nm~RR|4@@
zR#wNJ36UiVy)!d;+$P8wFu!tadzS#WtM*E$bKVr<EU`i!jgGURT}JIbQ+VFCFlQsz
zl)jkzEdajTF$WD_OGK{0INrz4igr1$B~$p5IQMKIDm5_FA6MvEu2+Z0a^L1L^R9+*
za!fA)b_VF%z9zusm2)!Xy;umZk4?s!=kPdHVTEAAtw@m5>pB8Hy>FQ)txOml-W24|
zVsc2k{O8=_t$JmY(K1#!uRw+ds(`V!5KEHI%%x*y5c!8tMroGp8_tS<igLy%sNmBk
z=L1ZYkX0oM_6r5Uiq07`WAh7mHm`MopzdLfeS}%&2I_<9IJg%3m*6NJoYNrI5eEpQ
z5jcCP#FZjk@j=DZ#E-4`nT`DOB3uv#VS@U#;?4ZK_LXtacZ$~;7$p0*3n6#b-ovrf
zg%ECHBI2}SL(#72z!>4q<6;5uf#HgU{trdDUK7wMHv4nJ$PbG=;*ftA!j;67Q@ms!
zc$z?e1RX*vz9pKENN3$*0}+=AppJ53z?gwLE`Je-^%JsWW^S?G=sUulzkH(^H^~2H
zfi6|@V6QQdyeTa8Hg4i8f5*~^P{_1z<9qyGIOi9BS>*PFelbT`U*PW7AMXfq1*9H`
zp6^3jWVDLTCz>1l0F*HSI|`OHv8hN(Q0LO~pXk|*m_m6Iy(1?^zAC-lvBJe9FyFbW
zx`h#_0DpqRUMV0nsQyE!Ys#(>v15BQbV-pnH+)HNGQUK+8<UdF?+T#;>{v0NAug$C
z+KhWlxY7R7A_E3Zo%Nh^8Hkw2)co@w@@$SLqm^_3^Cy9>4@yIYjaltSATBTCxfVJ)
zq@63&=~$L<R%ygFNCDHdaoI<yU>r-bG4?h`iF5-n#AptPg(bw~fs0@@PR}pFz!7vF
zZ;3=si8V=;j=3C)`f+FVQ2amyKcT5$)3+#73OroF$BExuY&H@-98E|#re6W-Iyc+g
z;G2}hy9o4*r7bkU9h3HNM6gSX!*8-r`ja3UGhHC6$f0jv36Pn?+2pt^H>)x4suGnE
z9ZGn<eUVVJY?Sya%$)ZL@fT3wq8Xu!3zD+8JMOuf$6QuaAzW}vd;eRY#-dp>WBniV
z3@`feritT3@B^V1W9Mslwp6z&oS%PP^b{{@n6jrSb@5L?oj<!s8bmUeKyf>{jeOu*
zlDJLjkPPNjW1g_0f9b*^88XnlW7|di;tC=uZ0Y0cLbC8D>*!Yn<i&y1VChaVT-C2D
z(s8bbYJ9N(iSano&r;VzAVmm7y}ud9iKM1I3)*q=4LrN1gE~-t|4*U$U^hXD*l*Ng
zT!0f0+Ls`AqFre@0g#48NcBd*Ec7+botjY>0vptc2V;>aHOfRhmVY2t3L!3+zRAfj
zl;`xdA<As#j7Mydie{&q3$(__Ec+DN@t6{=7O^?vQK4B93E)OwON{+b2~X*iq@F(Y
zXQX!PFkaC6`Pvb;f=!7Ou@Pvns|2}NoFhwTF4;TqHqbOs(3?*b;UrvKMEqM^EPCU?
z5~ZE6aD`nXM$Ig~ie$&fZ!f+ng|BR6HvBn|8(x8*YH>vgunFMc@Zn#OVv6V34X~U1
zB?u*wr$l^01T5ct^Eh9RW(p&Kj8&4NWV1Vf-Lkk(pd1rThUpYLsa<aV6^IMLH8W@)
zK76N>(5j5&;?1}a=gkZ|b%Ao6T*5WNMq`JHiy4;9cahq{qopl+b#@!O3Uoz?8=*1y
zuVpg!BdkCbwe;A#i)1>vpfs-uLB>YWV7B6_UuQ~Y|8ch{XDU}dI_6>bfVdM7Xe=5c
z=>)yC@i<e@BvyD$92r=l{!<97WaB(D=HClpHn7O%K%`v)>^>PE?k#~{9LAdQKnduU
z%7IwvH{{`(SE|W21NnPYpqba?ZiDOqZP;aRjrV95yI2Wx+1VjHBa(^_v&-XJby1dB
zsVil~_t)P7xx}2Rcg~Bg?*nj_3W5kbtG@$r>(>W{8d3Xw@e125<a@3A0W8w#%u^#M
z-Y!C646<B_qeQs>R9Q%~MB~Z<VeqwYU_7?@1J5qTL_?#}NdojX{t2=A15Qbd_UP6#
zHELt9&=)WM(G)jDVCyjNK`65hZI0~rNb;~ivM$_ko)rL#(lQ2%dXXB<`^V!|J?D8y
z7cwx+?f)nLj3CAFdZtkQZJgGBJw<5PU+p1&F^`gB{~MwyC3ZR}C7%+-C$u@}dDy=U
z<7Cl_V?~)S%mT<i0{Q$SVCDsiR#p#(Jz69}KBaG|#(*$?5F-lAVnzF<NmO8X4wMOK
zfZlk!q}vTz@wmms3s`D!uh@$M%sjdYiXC-@j>ig6%WA1IIJXpN5n)Es@c6OEi_EY>
zV+p*S0ChOdG}-ExKhr8GOy3d!BXvp<)1H9vhewFejk^#2o;wNjH`t14i+z9)vr3oy
zrg`;HVdT?5M<3%N_-2ZO7FUTfLkI}Z&)1iLJa`7;s3-k0W=I3^?o%c(53R&ie>TB=
zJ@{Y#rN~HDm9<zPfbTH04GqQxqEIcSgC2$|HljYb`M!Q|d8Gtlb56i5VeTpEb`|Lx
z+eV!dD~&3QZp9ClNXedKlAR{aaf{GA<qRv=YWzh|ZZ?i3kRq<w-SV%1=H4*v!<(Nj
z5lvtSWlIR%V1|P(wna{D7IG{Y6|VWTlQ}SWtoS!bm$j+zpXy;B6-FyiCDyPatZJRm
zHJwa>K8s!IS$=TYsbN!hVZC1vMR2gMm?hFxnKE2#$83RSnI~jURhx05P>+Y=xy+iR
z&b?SLCD9RZEEb_Y$Y!}6n>^>A*!B*`4kBC@dK3oQm~1~^K#Uz|J@BLvL<YkRon|&f
zS)gPiioKTr54986Je(mix1y}e<X!h$VRjzlsFkfozJ8i%`>+@J$^7PUlT&bR_0V<d
z33tb?4B|l+FOiNaF~7`;9bSfDAi)A4zM<zVUkXE}?flAr2QvrZNxi8xa{N&s<s|Tr
z(tP}t;yu}<Y2_4`33J_gi745QI|NX5RCR3K{w~VwV%AM(TpxKAh>~=2AHiyR_S-sy
zBLJ*`+-C9)M=Q=CakkJxc#|>UuLQXc3QCj~ahLrA%yp2JnUB677f5-u!WoLqUxR{2
za>8h!$J$Y}1W>E<`!m7zu`gZ`K`{qdk8b);=YcM)zZR3C_+=c^l~eW1`|PEOhB-yZ
zxqGpQTo~euIJ(fZe*wCB^ex>?t7ET(nhWWOFYZz@a_yC=$yTpJNR}u+7YM%LpD^Sd
zh@XhC95wrxO8#w9UhxcsCyS(<9n+`9srh*(mt|h8XZJJq6KS%uO~%P@@`H=4DaJaz
zb4ySb@mjAF152PA1dX2dU2&@*H%R|DdPt1`bN>TIJr>Bq<5CfRN0#@Bd%yaBf&9Tf
zbh{icGDtot5N=qbtZ_<!<$6dmwb5J93{ff8Wa_*_Ld?L?0Rof7B_)8j2(5Xe0O!Hl
zv_IY!?JxIW9Ii9-Y8>%4APmDPHy+b-F-%}{LxEWkzarR`=+Exb-~V3$u}otet;Hfi
z=0zQA#-yKmU6{YvBs}GCio<@yJAfW-F-hkHC+;bc)Ca5a{&$PaxcQK^HABuG!eFet
zI^u2-2ne@i5QTf4_n=%;R)vk&MYPL;Cz8y_`(kflye@aRXo_&ba21lWgW+)B|0_VO
zWXHL_V+u4+bT&M7S4T=~?&I!+E!YCRJT-@;W0p@XwOitBjV?1&&wLzf?!MSfg!#!e
z%k)VW4(|i9crqR0qQG$<x_2U{>IbAf`Q%pPdXav06rHSA)FVp+`NOJ5Up?AFTsB$z
z4#%;}0KlGpRL)wFEEMfm<R8yy{jv}<EExt!P(Sp3zm@hR3`pCEHqXZEqYj@C+Y5AW
z92p;sXGCT$pr%J~8_O05hrLy{-uIM<oVFPY#y&urJkO7_j_gT$%Y{O7DSeXs{ttq<
zEqZA-CIFTX+>k2RujBMhb$YZV+Fhu*(ATIiKV2>WNw0A=+R`85*>%DHxz6pBtX-EW
zL3V-zb%rV3n&h_>PNGB4R2IJ*-FNr6NE%f({_&yZAu<OwnW%65d<C%FMP<toca#7D
zi8$zMujnj>B(?jl1VQ#hKGR~iM`Y&ZIEzkhx)qW~6O{DYAg#1=0VzD3I^s-0QzfbB
z0@uD66h_@UXLNF>Q1QEZ4~ynz{6m<(HOO?$u5bBOKys&77+)4eLnR~=d*yqgAip<>
za*MY7#Ht`}3_PA?L8D8G4iE~flB(q;5ze|=(P3F7KK)@Jw+FGy#&BKZrH+pPI&*GS
zlk(%}jIfqt0Mo*L*^iROJ-^EJAhC;R=iQxxQQ|frNFARLW3l3D5Ue8@i?dgNJn`Gn
zB|IgfEiV;e4kE)uoJX9l5Jq9xjHg*_I9?axPvLr&G*CN)6VYW@3e-87{;m*<th{AL
zl1iD&BLw>ka!n>26p_|G2A0Pp?D8ld!_Ojt<^k5-TA1&)I*30@s0N}ZVn{5A!H{TC
z^xLW^+Ij{~4h!^5wRD$Nd{u<Su_v9w@A`3o%yq4*(pKppff91vrr3&#NPiABG`gi$
zv<0C7sZYdGYZN)4n~_!sp_nd^kD_tGW=n+MN0BHC*Z8j>iVK&%d`R5378J$h_$S`d
zvlDA%q=jSMwf*yKtfxMsXaCH?y{!%WnS#g*F-CE*NR)*SUTA8Nv)gZkdW;&PsoyRW
z6&sLdEL?{~Fh^0`v0&s%`E>!z97dG3LJ;@GP6Ay5H1JjJuy-z@bo&Km9u{Oq_qRvT
zKaLBajxEmnzoBP0y}a1O^f*UYmSU!uxV229n7e5(`L!^(I;e6yE`n-vm;n<y*MoAY
zI3w$Aad>)zAXSH$V98T@#wP&$367jb;uO&oX>Ql_m|Wk9X%J4M`rC1tfGl<CF0Yuj
z0f;%v3_`4mv<6uu)F0M-uhX3y1X<t<-(>pf7dHg<=d}!qwrID>1e()jH}cOMus8XS
z4^4xnvfREm9G@3qcJghkYxdf>0EhuLHf$g_iz23BYx`BDF`E$s6lfIUVPO;kH-%(6
z{9LA~H%onV=AvC~He4L~#@9C~5ZesLAB0#ew2)Oy2=0X2)ESczJ#r+r*bExx&c^)Y
zGCk+Em1g$Al4iTth-Eh~GL#p9baE)H1-kS}@YLq}2+KpsD4|DnuEJpTZNXkTB)4Pi
z%<PnxesCVd2-IceL;zG`*v_CcjCKEhTR^(PZ6a&2$o-xWzeL|j=h?C5mSF17avxZI
zD+se%aX=LB>F!VY_YvK#^pFVuPLHWJQJPIm2XgySY!+{)37{0+ED(^@_jly?B^XzU
z@cWE`Xg~U6#ZDkNm|PQ4imxr2*AvY~MWkDV{*0PXA!fG~C;?KtS46l{Bg9QlUK)=J
zbD`j)w6rq{08vKDrq2YR5j#6)#9Tc)HAb5(V)_NZb+lRBuDC?t>0|VRI|{aRj7mwT
zeY{YLgJA<#M2n#O_~N(P!|{nOlh&!n{vt^_z>X$v&%Ys9lasx%-YoxyMIVD*Cb(L;
z5({VZi`j-%M@1g_7YH(6WLbh56juB}AeHK1d%Wcw=hi!p2i6`UC=_2B!z1X+A{k&b
zY1ZOQQI-?ghY*-$afz^kWXy~=N`$#j*46!Uo&PAe2gSuA%)y}ww=l-z0w_rzD!av7
zJ0&}vO1ys?2)7oSFE%oq<&S*2z%UY8fW1PWF^VWXN<HQ)g=?!&3hfbTOinIA)CFkI
zw4^>&pjpK-dl=)4*l=4QH`*k|DCts#X3P-ko{Na|#KEG?1&r(ErEs#4+{M%(b!2nW
zc3{+tePeuA&+f7e%2QGjF9`PMuxTg!e+nb?q3uDP{h*xx#$rVw)JYa7c-1hhG1hD@
z(8br@R4cwMN=Va}ri6XV%toyF+F}Q1Mgy}FA(fUFj|+4KSPLpT&Tu>{Y-*?{siQG1
zfi^M9oNUqt?RZsbt(KY6j-(i{tL_Nyy5M52DJ6vXOT>(7V#mgPX=N%|g3ztDERj(h
zEp)msgzg}FnD}z}Gb=)rsqvVQ+?eC_L9}P_lbs8Qos@1cjx#<3WZ}T53%f`SD=!JO
z#4G6IKQrkSUo5{1zq;`;scOZlqWvLa{iPVyhm{b4gBW0|@v@*?OSXEc8yxvruq-xo
z9a4Qqi1XH=TihVpVv@X3*?Y)F`Q%-J5pXL6za`2A;EDvTtYh})3W(hx=K$3>QxI9_
zgYNi?Xbn95oW6d3Hwdj76cZ+fMA0LekJ%v8@Z<Ss<d~mN6XmK96^M?Ag5bx(GPem#
zsM~ZF39_`{k~h?xL?G+&SecLkLn_61T9DZw+fG#IEAI~CR>5*q63NWBfxygXMsjKu
zNACfYnTk1la!`0!n3FLDBMAh95=8AfwwT4?#vY%C@^~|vnEq`6&IP9z^e9ZJj|p>`
zaxlKT_XHu|nnpD;Mz_5E7l2$$bW_;B4HGM(BG9=dpCy<W*9mi(SaYQbc89%y7!3M}
zxwVU)^U4UL6ATX1S>Qs!E-0>n9N#=$0_dL6%r4xU6#kaXbyF0qZTBfK=1{B**bDAh
zVj3V>HgeC_7fI(1nwBdJHOvk*<E}Z4Gs3yu%S!3eYjLrF$p6dj3zQRaM>BKICxuvY
zIlF4c#-b^_Vuhx_lKW(UQHGAkwPjZeKmeNyxCQgcOki+{&7_E&7yH7Py&rUL3}zID
zx~sqB6xv&9P>cPY7qZfh7e!d`P_|>+gi#3V%mYAOi*B7Jv5#H?>T6W725K8}lQO!Y
z5R&KRe&wr}(`dMjg9J?tg*3(xlR0;+2z9}G=t;3iv>O&r6=r>TMYZEm!PLB0^X?i4
z7VqGBQ;jc|bhFLxBciiBX&)_x*TrQTt`+W$H$@{%v$>?^Ej^oARk@i|W9NfFvcSXn
zYFsFQ?+?`Qg4z7bCb$`>5|4;5ldyAZ)Zykcg2>PIBRclo>R=G($2eAP#Et^UPe+My
zYbFRyQ;&N^I6wAxbZ+EMkU1>Z=85DHoj|hemGo-~^LY8<B>Noc(ViCM&teKinw<_Q
zGG|Z78nqc`3!)@g9oOSY5e(x}HBjsa4uyn~nzquhIX(D~!WbFYG}0y3{|c0wkCUxR
zoLth}crqNf9menEj<&fH8&x38=MnC0=I7oLA$~Fj{&Bew7a1uf$Njtgg=$+G>z;Tb
z)4jSbf~H!ohu<l3t&a5+$`O%S@VUr<Iv*&6BA|(3!zjW+sHj3XBWv|@eht(DGXhuY
zx-I%qVP<-o%hSH`F@bKJJ~pFj+UErMoJEWL<X(k8H`}Cmr%WwvownG0y`+YFQdHOD
zN22^)W&l>W417Np=1+hm&F@`;sDYMT@v;ari0l8@^K8&(fb$(<sgc968DQ9uKV>^!
z5MZ8g{GjlKYxftKaA67h$n!FQT_`F;W|+qWctUS796nc)M0Shg2l$l|v8o{GiikY+
zDE8DR2BBR827dGo94rhu3CC#3hJG#4E)j#UqB348gdEd>*!?1$14>O)ADFic7jH12
zXhHXY5PEQ95^diI&t?j-j&bY%Rvo~zGeUKxD7Pg|l+FFF7nnr>%lLd+@vKl+rmw=Q
zA8bHpc}z-@I|VpFOa2$@jzYUC1MIdDxXpz4YseVhWO$jyF#_QyU0R0YYaI?r;XCHc
zj%j+%eMn>Z^f72yL65D(=S5gVdWjd5E<!wB80^tqrW}>%P;&Vukc-K!yZ}ECU>3^J
zU7G4yEg)Ao*~$^Gk^aBMIH=3dNsiWCcTRx#liDvQ1HPU~Fb~T~HAS0gGx(Zde?`k`
z-C(oId>~4%>o8(SL}nIch&S5MQ&}?ws7tK6z%MZCU^*+prD0{vX=t3Z1cEw-Uh_*K
zuB%qh)%d3Xc&mdNl=v$g;S8~3s>H@3TtX^6u22Def{+c{>t8L(MH-<e=xTy!AL$e<
zziH3YMEQGFcI7y<9aR9hkI`9{I~ssGbG+AxX(HVGY@u;T(h^}WfzE@&5DIam0GA81
z1C*AnxI>WJhZiXdT0mly90N+;qq4``PlVs<VJRr5r5_4%N99u0_(z#Sn+1*VJ0D95
zWDPxUT&ia`2$yhA#GRte?|k9kGRJ|q6x!Ob<kw2_{X*UTEQMGM*ls?-uoB5caJN*V
zDQQ3|X_{8~9_&viS=!ev5=^DgQ_hVoj)$O3oT%U$+m4!OYNS;q&HZ(eZlDxdN1E!p
zgt<5@x6_fL42ka+Oty^$3kKphCqTF~oZO_X>2FU2a_##xTfHqhw+j1`TCDk1XE0go
zAB(j!!F4zTv9X9!=qBR=0l8SLmKgFIadCcw)lDPjokRkE9~DFDMNL62Hsk#`X8vil
z@h8D~UCD4cM(Y!^vlUMWFFFG*I`NVaH%_h2=1NB=)mZstP*)2p)K+XF8a7~e#e^t=
zCQA3%$BczVQjGl;#4km=kb}DJoM`3ioC4&Ejn^l!ed7!33Uloljal_<cq)Jkf@_>k
zIkarrL?|tX8<})jr;Bv?b(W79ZTAfjL~&l%-1xqp-A`m9&Ver?bfrLlJh@eCZ2Pe=
zvxiVc=moUfCi_=~Q!_<%YR7WlgmO(R$<?R}GGB0i=#8(6c6qomOHOL%2_XwK2gLk-
zQWU!Q{)yP_^djjn`UB?OF9~sGg9>msF4`=qayo%EUHJ;VnUneWqY&Bwzt%>~KEnj6
zqgE^yk-Hs(7A4vI%mR^e6k)zl2)ydUu;6w*XSO6iTx9irVX8h$)`9rNSx!AZI1t-^
zi)V9|4WmpT7-shtD521Z`ArcnJUU;1a|O6S9Li`|{+^J`PgW7c!-^|%7X7W(Xf)2|
zcd@R&%|h-PQO;0XNba8bHh@d4$%X(EBXPnxU@jL14j6jL7~(#G{!X)j2_`j5kPDNJ
zu+fJpk&$kzbM879#2>`2fq7@8^8hT7avjo*rqp6(p)Lll{6o4EIF2t<qLE{roV2<3
z33Z0rDGtO3&j+A_SckC8Yl%WO#^%S}dW4hgYH$?4zy!_8Sp6Lcf32s%QcR=zp~9x7
z10shF%K$2YnYl#7W?>tHO6PGLv++1Wq>DpPWCDA9U4U6YbLgD;Jt3CJ9*zg&;0vMs
z^%TlY{pp*+s01R5ZtF%-&VyC1Opg-cJPwxve3kDOc{mh5>fgD-{1&!uM3Iljg}887
zqvE*pv;aQCp*d~;;zdxdvKI?dH@zaz^&3RpQu!XFn~M{a#&|C}7lI!Ponjz?v+?FJ
z=9C1ZN_)vEG>M<y`{4I^k0v3Y$Uv+w!k_16X3ohph}<L4m7u9)M844v{0_Ue{y12K
zf7W$H>|ig+KQl$mi7#Eu&lEo0KZ+(<WHA<0;;c)c>8c7!{cSzF3RR3?lG6PNVHA%A
z$ozO+1jR$Ei%C&ljP7wMAm7hUHyGc1O^9n@^Ac^VPcA{cY*`G+=Rbk40%r*{)vqWC
zWz$_Jl7Q7OgQl$<)0l&qE&&t<t1iy3MNlz<(#T6|)gSsd2J4A9RD=fWnVqpf&!m$I
znW6Xo{L>@@<3oBjGkS63i_XiTE!zwwI(LZk1o256J;r0P2(y|~c9~Z$cLj)Bm{m_b
zHSt)iBa~_({)~oiiL%IH`$s(wyAr@hHOxpX9_6Z}P3z-V@xxRDZ~?f!u+G(BW+rn@
za-!*r?jHkMBDt+To)pRVbR}O;yele8A_}_{5qphmz&w2yM%b*-HV|wE%-1G*P_#r-
zo(FO2PoP|IR#M37de;`YFwr}{Cl&mXV1E!5wPujvXbUqdaS}{}*X4r5m1IzNn<#&p
zz2iW9aS^mR$su}+tDLmj`uufeHsdYYuNhw!l)t8L$>He`VP+C5KPjs@cbG5GIZ9L+
zQzcZF3gx@(P~{&Nn_ds?N>V2V&yxkXV4R{3#ZPX4c1?&+pUmtx{wa{<0F9*vqvwVA
zCAOfw(RX9<5(WSm*fH`5$_+_$B+5uvd0wC^BWn>hox|~ViB(#xSu~}sbW@QZQzl^+
z$D+H0I6qDXiMI&yCs0Y~E?8P(zncN6p!9voCvMlD0Z`U_Q%5WiojS{mSos#7T{8t3
zVHbIz5LcfL!CkA(7%UTE!buGAIN??jxf*l>8je&6b74>|$U6@3<T4>^pK98&EEMK;
z>Q+pgCRQ)~af47wsUYQ=#r`N7={dYoV-dH`ZBQ<!P6U{&J}Ja4Fv8#!dy96FWU`*N
zk%tJ&O(}~P#NlUxsJyO`xFbK$#m3;k+rcds6;=WD_`=VfP^zPKe#x`OY@JhWEy6`X
zAu&X++NMNU5b2EJz!E`G#<_VsF2vj+CW<uUdhrfG>WKPd!`X4^ub`N>yC>rAJ9%_v
z*ukV-SFCpzn7>PwsA1zQAzp1tn3`?e=;hRThj4!jl__Jl93MOH2BhL$Gp5HvJ?Fm1
z`hC8wAHOYBf|Iu?ajgJG9o>@-NB)?9r?iTGQj1-G&F`=!UBY?FJrLA|@fbGWE^2DX
z(>}q$xVS3%UVfk!D4%F1xvMM?l1AJfoa97k%otNI+vqo@SmjWy{Y04oX!6;A#o<C+
z4;+hh4WaaE#|h1?j#s^m<);4@D6h*|wv8#UpLFgk-Useh=Mqk&a1{Z57q+3GX~ZW4
zS<tW<!yHPqxjU&Y#=t_f!ZIV`1j6ohe^;dPrSdwRzDyvW;~a;3s~PUL`aJ-R(%CsD
zeyC@-T^UXn2tFawpT$nQ5zpkL^JdPBcl4awkBOA%>~y$zPpHezvY3NMVyoX@KvFnJ
zG}jPBzMKq=ARTeEXmgbs6ILK@6heaxXdU(+5tMM8a4j4CfoFd}_7C!~J4+CS#wkhL
zz<-If+%`FhMaM^Uya&L{V@1EJ#Q}dTfM$w%u-t<Nyc9;ox4HnylDbLE%s(S{oG4=#
zyuSdK1Ib`z-i$s$ZWN+F<?eq=&PKCaCEhCE;@ECdyU~sQ#9J<|%qn#wOzb6$+N188
z9!HAsmk`)d9Y(D2Pyu1**4Avab_ualmPJ)m1-XUjz>4Ic9(0FLiMni2H+&c(i!*NN
zi2HUz+~nFa$5%_5ecMpnk!hCD>1b`QM@a8UzCyI!?#ci+><q-wf}PC*7A<k7XujGx
zkMpsO9)-vw3+ARAxNw9Zj5eOd&`{O0YuwFEiP~^ID#!)J@dAmDC(BGIH|~OrWv>hK
z5~bO$#zz+yFG+pQOI<>;^l<jACfQ{Ps5T}$v8a_0(XI&a(2Qpu<3$V51hYyjKJ~cs
zPKOqgnk-g%qDa!xN-n-Az&yYxVL^<FcC~dc9i!xMyjWtGxG<hHh>{`9?Z$#6g}>#v
zYr`i&;UGJ~YFr~CXNBG;=>j%<3QYZY=B(IF&t^oM7>Lh_r1Z%mOEb`+e|FkF?42Ic
zvs)0pGVppNzWlGid~b4`rN#bFLzr(w5kebZiIyN20j7;4gVpPVxs`B#W=9ln{LQI2
z{-{@bbqx$>%g=yOILxod>57+$G;5RUHLexloVmJ?cuS}Qo)+e@T)nMQ+-q^uv;2@{
zB}Im+#@_^)@2HQvqwBdMKXl0~6A<&+!pvhrx1jlxy`E8L#{4^hzIaI}?URwkt@W;J
z*-ZdMq<v6`dp-w&h?+Xu!vYyVQtXJ>`}raX0nL*mG2vZa0CY*a<+{Dpi_kRTAS2*?
zdd_*$n{hLx*l*?py<Eg}<V(fNS^>v3qWwa0o=q`*|6GC`nJY3D!kj?2cydTk<=KiY
z0drwwnk{9(kl;MftCcnJH{!3#*o{QM(jlUNRO4x3)Hq!y6)%YLe5X?$2k9o&w3mxF
zbt0qYjO&G21}bBNxa@u4?**V(Db4tjfGiZqTUwRB?-elEHNrKDbM=h44iTm_wiB_$
zqC`$Y68q2O7dmE4i`Vq*YV{IIHah^X_$nx}i3Uc-IV=1F(hWq+z@}-xl;X9p1V)#3
zic(o1SoF&>Vcpk2C|M6m$o2K?c9~!e!S0%oYCnONG?<@mYW&qdf#4p(fmh{fuS2;9
zV-p&~B3=Ckx2{waV&>n%oUu+L`xX$_CFZ<QfI6{jrIkSf-QYcKjTfwe8GPm{t;;=D
z(^cBB=-(hNmM+EBisyDg6ohbHn)kl?CL|nToM5URi<3mVoCraz_a8_qK1w9fww~Rn
zia6egy9KxsY!kXUENI3j{|iQ~Q7tj?%@#!~;+ixN+le4+o$JvP_sA(HvD4&UO;OH^
zGaWovtDN5ob2TTWPU3)koE6WyZxu<2f>0mQVr*Yw<{hGscb<F!Zh5-^nJ~en%YCZ^
z*@&S4lc~$2|0AWVmPWCd^$q~lkVjxG?h@sKa3rb7TIrPkjCX-uW<owz2C>7Bgk>h-
zWZsU;mB=E-N&HyLhh*=TuM%j=s)`T1M@kE4$Ly{-aiM6nl&mer;$#0;{ESrM&MMli
zjbk0V<2XYIjWtFL{NL)C3Mjf?CH^HUiyNz|WEXn$QoB#R&ncrWxL;;z5Em8JumRE)
z3bDOVvo;$UGq&tdCQa*3d#fKMG<O^d&8T<4G9VPGvukD?s%MJClA5EguZTi4*ow5{
z=KRw{n!-EY&rkkhiYc4+U4IqG+X^5*8LKS|#lB{uH#XBFpU}3qqGuPNSsBr$_edeR
zjruvhlFihwh4JY=9bm8c0SI@IrVN1j{p*5M=;_nq4|>jhiXAunNQk+FROuv0M^pb&
zI*W)6kkr`-gIVUT^9tE=B*-O7k^VK1t*{(#=Tg$klY8l>h50)u*a?x>ka5R;Wl99c
z=PJX(Tm_;jas<+j6+Q^$V)yBW&9PYJLm=?4M)cFMctjLctatasoy+qKJ2-_SB$^1n
zNDZ`8|DO=VGF6LF=6x%eX6f9aTU9?QI`drCg_YPqh--slmJ1ljWlLf17ucgA8NC8%
zWX&bW05dGLo4F#OI6t#1_SbXnlN_|SqF;bNG>MLwTGUHK*FK@xIZ}w3#`Z;C#A5F)
zWxDEEavZx$h*_3YdVG8Nm5NtcEF|}T2IWr+r9^`$@bA^LON1H&MevIm;L0CGJ{pem
zS0;tKFyn12{wmr<PWKhn<7FWpU;A-RTz3`7Tn-jfQsC__$lP<tP)hM-VgB;r%xv}e
ztpHlHhr_wy_*40cwiM~~=`X@u6oT$@;mE4co<n8hL{v?$3l9)(!6mFEvm!7oOR)PO
zt3k?zJ|+%Q+T2&Pu4KpYWkK*qE=dD=_B%-C@Obh)JzSuRPW%w&+~dnk7)(=ii60B{
zYLN33WI+Zvn$fP;4;nJqMb_e<qRdSxB@pvHJ`9p2U-#2dh@Dp}645s$`BfpvH<HLQ
zSA^Mu=Vdj1A=<^`YBq+V6#~HQXG-*EwkR<MdRFJmqqBwOrnN?WpO1mK`h023ftn5w
z==>SCC{3kICEq+;3}Ecn1m+%-%r=q7{|HluWieQdFRY%aNyW$=_&8MlxGWZX;?#`b
zC?U-T=l1^-;ym$#vyavEHNdF7mJ(>PW4kp?)0Y((EIuPzQ;zN#<u-GXa1LGhj5sX+
zMp`U_;#QGnJ(@#>q>V=fQCGrUAucrE;93TFrLLohXbM1D<`HD<COz{l?N4ajjn;;A
zelnUt^AdZP5J5*!?exd7>ypZqf_Gk8y)JBu#mG}kTMxhuK`B@ZsJ)I62s=|a*|<<t
z9w)r>Ro8n&sLR06P8ef?T9%Mz?>1vh?<bs7w}zb9e0?Z41)4kzLt;*eNTg`I=Y{xd
zcuQ1?{?pGoregz8iz0@gt=L+$<W%Q^2a3p&O^?^g<u`(07%F_3HBH+H5?<=k<va9D
z<(SY|%lA)%GDA3>W^y`706%y4>)3z(#!%*=RC#HOe1tHwjMLs6dhj0tU2HW8TEBj7
zo!Vp*P^RAbJ@L98U3Z0h;Y+J*3gWz_Fs0uT#Y7kdB|KH!t7p|w*JCcVS@AlGV(w0h
z4+_aeQ(QoeM5}KB=Gw^CxD^KpV6<RVk~`BFoy|MutMo(Wnb!L(QI~*pWjMNr_?=Qw
zoX)vju~^Uk9%i)dbZq_P61iX^Cr!`_TawIUB!T(ZtDxjS%ea|fHz2#YMzlmbw-gIZ
zd$XSibD0RhO<eC<?6Oq>p><AHE{6&-Ba>9MW4%v-z$liB&A3v8Sun&EyUlo4fD6>D
z%x`E-Dfu1jD6+vSzmFN}NFljBu**(ckzWXNZd%^9<Hk;aJa7?Zh?b_+R-RF$nq)%3
z^`-<kR-$1}F99xsVRv<=fGK_u0QSa`ZdGvhd%b)?2MLWheP;0ub|-|e__YA%p!4c@
zwM-yC^r46@5}2Lx4AJ`J96=Ip0?+kF|17A?NiRVk5aL&ee9XBKo}RM{NKt5MgjjhF
zka<9LRX>($>j-plx*KScHxTX25m@?mbO|vpbD+n00PZscN!Au!$FDC6_2kQe(Inrb
zD<O+3Z(ZO-eq+VGz?1S;!rg3|!qA{j#@ubdax0Cp9ZDnYega)NyuH$Ncd?+%fkwJ^
z;jwKCgh)d(r+39mLb5=iqayajc103w&^0ApRU)lgKh{!lwGcWEe*5u`o*9d<b)W~v
zyCQS+|0UHq&Dd&telb4?(~mn0jf9J=P?xc;qj{|Jgdp!&LAByY5qzm*&a7Ch=iK7l
zwICOwJ9jAFX0Xya;I<t>XsAK<{c=C(*vT)USz@n)!U_GNAjE`|UTj^S6GgGKT!Q7@
z_KT?R3~ILba@|#Gx=~^Nz7(YLa6VFy#iDPdyFY$d(j#*J82=2vo4Z_Z&xG`Q0d9W9
z$|`hnAKRtKTcLK;sV@2~P!=&Ph-FS7B!6vucr?DaD>S*Yvi64JasjjjbGokdS(JZg
z(xor61^S%;4R|x=#AAB45OH)zcohxMZwmA`y{CxnK35`Wf@tBd3!!}mSk}ciMDRro
zJgfnKB+@M6OfoGWci*jmIEqVAguW%nT;ZGo9pJSBs2GjJwu>&fdyxR6bLNK=1-P&Z
z`RX&hlLXR^#8c+D=(0@N8K!Fwexs&}y0nv?EqLj3O$tkdC!x9it$wt+lAeDE)Wph)
zmX*SS&qJCWj0{o_yeq^N<c^(Q8Pu=1r{96gy=<^ML^=f>GaWY05aa?-!70CkMwt*l
zA0p6UJ|T!w&tuQO^cQ$GlL+v|DsDo6CQR*j_S?%TJ7;#pH}y;@YbdB%I-8DH^RM$d
zJ7a^r`PGsvZ!re`c5E%o1=1<CkF(=5Liqs2Zp2STP!z1q<Dz|vEYOMI0!IPv6y|C&
zh^2k}V}d*tNrf~ObLdsUW@4+KF|QTNeGw#&5u>bO&@iC8)r#U%`h^w6=wKYEN0+-P
z|1&ku>B4e%(ySKemjJmij4r(|Df}_+UsAY)6@<8;SVysMuf*DdGHbanp%N>9sd$Uc
z8A_)JA^tj{*-;oF4c!M6uPIJ_J022X5yI0<Ycam}jzAdKK`6TY4&)h*a?*!2^Fa`1
zkH(tuY-hbdFzI{HXFsZE7YwsV&LS@OatUCy-;M_a_>;+0Pa!4aHG!@o7JzJhs<H3E
zKvN%+2`vVs4CTXu`B;O~$&2)CR`qkVgN|<5Lkfhxi-IJ!7UBYN8!r5Pp-e(Ql+n%W
zCBjH4O~<Z>7B8_iV-?yfAdg!dpT*p-K*JVfuM&rdaN)b*phmv)1v!21KreSb9|nz7
zd}>atp+|nIFt)c*7M>xR3Uy7N7VB1cc21n+5fZ2#8<Y@}8@7*xof70~6A@f0>bHdC
z@j)9i4(sScBi*131)S$-uX>h9aSqqwOZ5y_FlojKg&J>O1)x;2zfthRtwd)5t8g_}
z>@6S*p^k=0Gb)G-$E)Kpp5JE(K}Ft(--xD+jN`=np?Ac6LM)Fc_PKSCebVVQFn=XQ
zVvKVHI1v}B%b2<uuL#RuCibxIt9(rmHREeD=go*O^g=R>YAza%GeqSUZwz<0<E#>Z
zg*dO=(pTij29(|5So}f|4a)AYBl`P`-?gpP^7w}XK;lnx3|E4I7$1a&<J!2<z-(%C
zta7VR%Pe6c#t9xe#=qgI6$Q2v?8mUi)-ab|aNP*V$PS=l$Sk0@*9QoC8n-E5_ZeAC
zY0~YBI|Rc1S)E<+ks+Qf5}K6J>}@L~50D%b_HdmDr-#=FrJ5CXPjOfI5;HbihWObq
zKf_n-Br#@+4~{_cg;7SFb@e<Y5OU7kxCDl@NsoaNpr>Iywr-tQa^>V(N|`qj<Wk`g
znLJKGH!TxZYdR<Aav(vjkaS<P&_^2u(yahmMEnrq{Izsb{zHP??>HeCiZex1Ar2L!
zJ^7<ZSAtQLN-T3YfIriXLKSn~RfL$;oC`9|YVV&we`%u*2fFJo)3GPlSs?j;Lm;1H
z$j~9{b7PRXfLe#s&sJ*|5I)vPeK$wYl+33K_V6ccfn`4Fl2Na)4;7k4Ls#Oc#=jKi
z8guMD%Jm!t&9VMKG)23H!@A<K6;EXZjZ)7L9>2BtKOyv=o_@`X_e?+`BaOb{@pwrD
zA~3IOcD#Q+&n{GjC}y(#Vif3TgSt?%+0$lmNsQJ*7fZSsN@r$Bomk_$-AlYGrM>3Q
z9}45^1g>qwZ6c@~F#_o`{qaKnk)22_R$5SG$W|y>C2uGQ{&Q}pA?G-euDjwnQTGdl
zP@Rrh)8ZpXI59S>qpSlzn@Pr$CNt8FBGpsmM1PM6uX$DJu_=3n2ZdXnF#lv>zVVR&
zW(I;{NjXlK6UlHB^H~AvR(O|QbQI6Ih{Nn@N8$rVgHQ*RW4tHU5a}F;6o5!VG$M=)
zy0{o1jwzD&va!8DWX_MVmrdlALb8Bhy&a!9)-R8Yj6{Ea?&PAqwx0dn9#%}}iF7|;
zZB*-(V`BVOU~X%=DQ5+EY#tBhb|ug(8#kOC1X)n%&#X(Cx!X&ktOJN6pIpCET3(lL
zEmNlsC7vvEB!(KXuv+n~AXkg)M<n-emPj9}W&8o(72?7nFUqVhwm$(3PIY$9j=S{C
z9Nf8iJgUcBz=39;E&w3$vL}N2BRYhFo9BEL#JuaTU{m`e(f$Npfvo6w>uF(pLR-<1
zlZy01DC?pwnvT^<>-*Eb28BrV%d+%B5oViiv#d1w<2pgP0@NEne?n!Q3})d;YM3};
zAplvj8bZR)6vfBqP4A3r^*kkK6Sx?^B!JRmZJD-4?~3O8tZTV@=@bZ87K6+N5lTKO
z#FfS9U?9ds=Y}QbhU$96l1bRSE-I7MdRQ-`az6D`-l7updo;C3Aid%=Va}O-;ZR&6
z+9jn!V13ec8i0kZk~BNgGn^yPC6ql7?EJY9RY<!m?5!^QMv;$BG4aLtCIIZ<M1VU#
zcy=3WP$kTuR9MHCsBd4zE~pug3(f7SEwvV!bIt&x`q=<O(|eyFm#EIN0c0R2qO-EV
z#(1k~2rlDpGY}2^?u-%AzNBh8xy(g@k5TL`EOQy@ab%I7m5I=>N_l&?5b8FED^EUp
zW|1v13nZ)6=o02JibY9_?g`WAO>=}JFl?MqS|5-pT{YZ~Uy6TI7C0T81x3kO8w^BE
zM8OhWSHVRN!pu_kJQMiENGLWv+pl4rU5{-<SZFjNDz3;Lf+#=@cgb4(Sd?Xfz{r(Y
z^V<cDg9sMbqNl#ggreY;JC+y&JvBq}6Xs)<6qW?*DBGVc@k2N9`58WP`0+0)7pf<3
z>G+Zems1hwG(H_tLbB}A+_v61q_Uu*LSs+4r2v05T^mxwAHQ5b&^kjD_v!pRZ^rD{
z_*`&HHpbFpcv@m1RToI&4o)IY$j=oHFIPAp+;vyb>|~DpNnw<l+az%^7NHk$wYyu-
zu(e|gCYtz-s4VT+q@bJCx@G+fKrOIr99Zbb6++04(8Z1T&V>*z9J(IuUr}uSS)kuh
zKoVy2-+vdxT&N7OQjFOb6-g>1%rZl<%=bXdPOd^7(Am}Ug7^{^29>y01O=bqnD7}r
zJ3AbhFymY2`%Z~{_DFnE1ivD~tQ+Wu8w;SG+<?bnQI{weSyy~&U(2mn0?jvW@Wii#
zrknc%usnLuePG6^LR`icwbhV!pCHOOoxgZQ1ZC`;IU`oRm}g5V%Y%XVh-kl?t`wo`
zpDsxKO3^Rli<dyT2;Ewp$}jY2VJ-sO2m)Wk`9j=E+CY!hWb++=7V6ALF-xWVhFncK
zGbUsGCBcgP)j5Hz#9#C>5-E)-Dht|PI}yJ1+>&2sDrv{_LR>~p0>&g_G4oO|ReDZW
z?4{?-AhtD{0Q&^NAnXD#U>8A+*<8{tr{td|weP-Igxi!}jHI4*836hA(8XWYbN&(Q
z!@gMhhXCZNa6a*45w2>hHouixm77igT@)5>$v<M19|4)0ItZeMtCom%?Px)MB*b6h
zB$4=`e-OZz6j>qua5+R?#SNn)8OYXzt6fp#tjn|+T(-Uv#LeI6Zq(xUqO&lOf<Cdu
zRUp)_WA2=INYDPN?low{Qv#fvFj%9s((`2!g-mM1@>l!)p4w1U^z2=uPTgw650;j)
zCD_?=2+4`a*95@-WYUYv3lwFfBd0RoDmv$aLVwDF`whWf)@?*cyVb6Noa#S>9ZXA@
zwM(!vIiz6+>HV!h^MHY&5)YR&&hoW*RkSN9haGnFYhDXt=}s4dvO=M;HxTNkWb7ay
zd@BwSB%xz~JVMW|B_^0?$gW=GALn&-#va!d?-D5^d0)>LWFh8MmzV`uaW53+Jerjj
z3xg*FxNKYtz<3l-l?Y3Pev|+&3ZW1ar2p9Uex8SU>ad>SazCn|R=n>9C<F){Mk}U?
zFfUuRzUUBX4zWJM%ujTZiP`apo?QZlu_1QDyZscz(#MclZ(=!y*Ge<iyAeE(9GtP>
z&5Bu`Fv@_<2SeE|qFg;@NX%P*D8QL_<0e3o<!=IUQBY~IH5To8ieZ8Fnv(T~exN=$
z*Awlxd<(%Fh?_;b0d@M)tfiIq7j6b+-Q06{9HK{mm@WAjLi(|vIX~%T<C#n(pS(9k
z<k5zxKS^cryIa6q6VxhPO|s0b04_Jf##rnn+6k)TSQAIBOwchK1D*LLtssMOr)cU!
zT%Jn&T7-pL$My&}<MjTw0b2fuN<;U;(NmDC(M%znVzf+z$$dJQxl54CuN7EQ*L>sl
z;zjgx@;cu3=ODEC42*n+^z6@%>Lw-@fR%VppueR%P8m4=@(U0O<WRv$Z}f~Ha_O2o
zJ!ag&vm1ff&}s$gfOim@^G>HLvIpB!AeDzRb7Grc74Hos^#ke^<{+Ws&oEQ+oV!dk
zbZ@~qnZP^|<KnydB}<edj||5)LJ&cWu(({ZD$1YNAz89Xjm>`z=#ruVuhte|soFH^
zCPap+1M#Agxd9k&P|Xj;o%a-w<VXwi2L(~kNv7tQc`pR~7{{%Bzx=E|Th}v%B6ti+
z*%L(Z3YyY^_+BRP@$8BNe#0*mmj#5*#Q2?o=njD{80smsrkDLTzt1u@{w~7Jt=&aW
z1L0f!K4-(~UJ>O8uTM791-d#qmX{Us5)iR@Qi!_!mCAxWq~qjGeg~Z;pH=yAY$1TE
z=(wA^Vnmt$=#J!h^Qqs1P&wjn4aIRHNQ~JXs<{7&GT*s0kLF9trsn;iQxd{nCpqSw
z@CUz`B8$a0MZs2r&BuB97nVy%#<QYayuOOOn*W^v0?}Xzi}rY>2S`DAv@VF1Mfh8s
zoAmVeph=3=1m=#$P_0>VEkWuX>{+(dbMES>DB}Rp60mWm;y6PTLo0SEI@G>Sr1_zX
zKj}3$3vmf`uGbeg{1HHTcg&7E^qj?4tCL3jpAaXn;`>pJr5^;~18fr&c5yF}REKVj
z8d?NJlhRW|#_T^qPf2XL)G2ll#Me6J&W)2E^7DA#KrGaAt~5bTn2BTchk=TN3wX!+
zjqSxJ1oOoq8I0~E!UY~=eq!TveTkr>CQxJCBg9!VVPFM?(b!ZTidMDPBhG`BGCF&)
z<SRlvKlPwn8ho_)2ofzz(}hCJ6`duph%1wL(1^Q)xS6<uhC@ctE(KQzGk;g({>4SU
z!~xOt^OBG}-Jq^a8?LkgdHJ#8Z6Y2KY!l0HLGEZ6pQKAb<2_-PXqFPJ3Yo)Narxur
z%j^>*c=rf$1*ESfe#;X8F0M)!A=<^on}QLzC)O84)6K(dchx70pEa~<Rkgk#n3n8L
zdScn9AoCaGQ4k*%;HKg%bd12H>j?2T382)9Ipr&Cmk{QI%2zP<oM3e672*y<Bo7^o
z!L=&|@?|aWk-{tg4C$hy48z-ni?vIrLNBQ!xQhk3IBFlpj&=S5V&*kjXw<rOD_^WH
z)VcN5SSBGJn+eLVa%M=ewiQANe5w&UXOi)50!`tYB9T}E^VH+mzd~55H6*Y~PV>ug
zg58?zI^};ubWdTfESoIN3@me&dm7MuWh=_5+dcy5|MX^t|8qsTxLB4a!<K6WIhz!;
zI#~t1B9Jl-aGVie`x}Hmj-5*tm0UB<5SAMU6?vQ^+T78KD~0f>#=+0zZ0ZbLH4)Ax
z#Vq7TtGHhnLNk*Y`KIT1cF_qF2QQu$;1{$_C15?CBhQ1m?=_n!f<^irosNveR|UX)
zMq&<(Qoto4lr#k*V1%3bBD5EXBtj;1yvnlrm%=k2h#JNag&5pjf?Aqbx%E&9X6dG2
zYre$m=B$i?BpW?~U?Tb`E<qOIlIldJ9ml*}yny>LqZcuH1kqLvHg^+o(%<vzlG?LH
znBVDk;(z8JaRuc{pI3^k=+{hD1F?}H_#j6kJjf82E+NigQ1i&X<x7m^1XO9qRj(HL
zaFZADerj=(APaMS9Hr1e+$+Qt=^kst(3f)(R={~+|M5Ra;)Y`c#YhSEUMtM6)lxvS
zFTMug!K_MPY7EF|3u)5o`oVAO%#VK7iZg}zZIt^gQ64H2r5iM~czZ<{!qko9$s2lx
z_uW#$uJo_sZ8RC2%gz$u#?p3PQAyT#-7m1PnHfil;CJk`bU&-g{}VwL4;)r;0ns8X
zfi6}n-7Ro)nF?J5mOrY{vTyLJ`!vTZMWB^y1XD!D47@EwAlGaLWFflkn~+qcLM-G<
z{=;)Ftn8|hhQ0p_WErOXSnF#U%$KeZp4)~6UNvsdKr&5`_1J>~_~-~h*5X|e&UuvC
z0HtwU_7;%)u{>_lgjkKW-ky4kqCWLM2o3a{W;~;3S1qlgQ#9dr?D`IH9s-HuM_+zY
zh(E>JBP~T=5>yU#s2b8;4ePuMO!l2~I$}c4a0Ocs%^^pMr0@i|8;`Gvph0x)IyY8G
z15V_4f&Pq6Z`DRWEg?n`LWyB?{vN65XJ|7>aG5B-gVKpbA?F~1%mtZDF+XYEU-$n&
z6YtpiZITJ(Wp?bD2|4M-kO)^EHJPT<mJq(jW{h3%flKW%^?Os{*~a22A#OIrjV|@7
z0Qc2uWuhH_dmpsR(}yV?K}>3lbr&xU>alQwBfU38QoNd09Lq1`9N2K`>cH4Xw8a<y
z7}f&~&cH4fNV#P0h8u?{e?*<4Av60X*RT{Q(jdQ2O*3l?<xS*{see0B<{;Bh>Igds
zDtE@ZKGxp*|10y;*%Yf<v9^0zl2i6k9Ejos5w2I2JGGAdfXQyJ*jHT+GV_An1FJr1
zaV{=VZUSjF8?ppi5=ELBVAA$-g%6TL?#djcv6!R;8wup23i>XaP?0o=tO3)0WI%vh
zbu<O*jfFy}bydTwx+O-U9|Cf_VgJNcIHLU$olEMTf0ijm6{~OC<w-%sxyNfDUKRmY
zd6z57-nRl29bo}3eS>=D7i)@Vtq5fXwU{Wb7n$1@%^GtjEp&@O-X3nuk6)At?2)6%
z=75z*;6{`6e+3Jh5kVB6UY`X1f=sheA$-z!Tq`<H$P71Q+8jSrX2TgG*OfNojq)}8
z#X0a>*+m&bQ+S4+b7vdivm}315G?4I9puuhK$v6LUSm8Es|%s(Ow`EnQKHOYxs%2{
zq9yclO?abxnV3s*E&TMV{0yfQkQjM*Qe@6uQyxodA+$4^TTMRISoOmI@U4SsV4<E}
zU)l=$0J6A5n6t1NDGvJxh}qATD}DV6kQyfmq+E#Nc>G9&->FrJPg{-Gg_sS!x?qE$
z>jNJJ%FIQ}oVvxJFlx)UQ}B$tL{lAPQei}wS<OGwJ{Z^#eONRf9jFq2w^syw)0L{#
zm@mqMt@PY!$%8mvmUF;7k=cGMej}RN&%hq=RXvm25T5MuY(SXrIBZQ6ba(bVpZ76-
z&x1#@2TyU9ek9b9UejE{*vjfK9k{%vpHO!#n8Z(oM@ouMF$&MyMVVP_r{wAWgb<fq
ztNJvOFbS^ralgiSJzBzzMfo+3F|`ZZOo&HSOzz;pNo5lGqO+zyU5J}*f(|U(on|~I
z5GLb^ubFGPHOh?HBD0hm5~7aD6#9^!DLdAuwBc)_U|CBW$F0_c@b@d-+(j8D2ymkl
z9ym4dX@cD797W4DkC<^~vPQMd?1Qp)Es~i7s5x<mr~T-A33i<sUKF8g=e0@b3Uy=2
z(O|~ui5CS@-gy}F?z0Zheg*Bd#>ieFE++DgzUHbj2^$dhVsWn!H!l8OQ?m1jQ1z6V
z(_-Isd8Xo=&T<K>2p2*dB3+h99Ig_q)dw5xG!kDS5bkO9%Wb=&%nYI_upnic{mOb^
zE-yZlGFZG^h`5i7`0w;=zA$C6#bJ)oSpEs%%z5lDS$qocY}7X{2Ntas<a~FXesIkc
zI9O*6eBwr-)CQ-wM%*WY+Vm{YbzvB1>&$<>^#L;*@tfuovxMZv&6DJ31vztkF$p@z
z`lcC|2vz17#Kj&PI-hiYnzr)a6+-no2_d%cMm!^lDYkhtVj86R!%-W<FVeYFi7`Z>
z+okxo2y@}+``9<e8lTLWcFv8p@-r6tagLsyG6e~&#@QQ#@JZQGAz^om%=Kh_#+`%r
z3&{hp;?*9u33MLGP)X>J?o2_ns=`BI;UbFq>X!6)Mg+AR8($D>Z|Y>Sh8>HuMfpp;
zI%TTGa+`rzj1<epBEN+|aZl$^KNn$!q__q+>%?8cTppAjq~dU4^8y;HF_^JXUao|6
zjf<j0e?|~OBlDkrxrB`f@b@JfTC@`NQJ{;NV$Y1m<y(Nb5cmnmTVW{P6!!leLP#)|
z-4dMY&zn9YcGt7t!$ggcZTx<)Fu2cvJ~rAH!y^5$@j8|&80zN?Iy$DsZF<fU$6Q%w
z0zYA^;{A;}I^$-Me6C~mw3zlOo;8(7kw`_lM7!&-h=U29j2{r}PbwlQ61G@~I};cF
z(uS{;Npy~e6t6U$B)KFE{ut`T#|6<&Xp08oI1$d2QBDS2Hwtk@N7#?uE7C%x_#d%l
z2eiM57YWwPLnY7{mfWAz={LF+29;j*J<-mvhLO@TGaw~`$tQ6yQDjHJbT2|TnCYL{
zcq77_it^X`YV0IpOhA?^-883o|0kAMvY1tKJt@pQEG`73?#~LI+MUVEH+$%u*j0Ro
z^&OhG^=1LM+T0>D5c`QXpBQ!(VrG#La~4*&<VCR3Y#=zxedPG%i6Dno+cC(G6%8A*
z8{Hp8!Uk>~z^rfEIersSuEp^p{B2fN7{`eAw{e<c;*DE{@N&l%b2zsDl_*y#EqK}K
zrS;NQbHUw9QWUim8T1Z;PK;NtcF@w!w&L%?T^)A9Rq4k5DahZ|6>bRe(epq&SSQCz
zA0H$9|I-iDyf#7k=WY#=hkg##5W}<_-DwFJOO=Yk@Y7O<ZbNFCm_0-VgAe@I1mw<*
za*@eji*tc+Pj_^)J=*xwkbI9#Pc^<Lg1$6+=Ja??&wiQxqGmEW)HT}{Am-`)!x$Zm
z#5)4j04#f|@uBUYELW_zTC(7XhOo?cPT~-s#oHGM3Mx!$V}%_+C`bycw3`UeDw^=n
z7N~*;3FSj0HKa8@v?CN18Neu%ee;VVS!*3maQ2;ebkXH_ulVE5ctKz;SGoX`4z=>m
zMZ!9JX2t00Gw6gI_ee>HON0`-0^~NpbxPX{aVxVmS6r3&T$x*=JNd~|(d|A%I;w?X
zV1ImU7YHgqOe_v84-}PqhR!X1BHABj-BiVkZzMMVESNu|I3+UUyHrrFoh_#TpDWWb
ziYMpHopvQ1jft&JoRy!)7&dOwbMd~8C@JPGIROFc;)9<ffn}2(nj)*%W_W%5U@p|q
zld%}zNQjF(ULUSZruc8ds4*Hqt|lxKu=Gneuf-uHHjK_c87?mr=HW@&WHgw&0Z@7r
zYjL`sv-s#tuO7b?Vz#7<dFePy?Os5bgmSD?i;t90pKPG$iB|}6fvc!=wbG6ogt?bB
z+gR+OUX;Ppi-M^K!O7;tn<88UmduD`JiEu#&p6}F`aA?IqnF6apKP&KlL_+q`eAB$
zgyZy3S`yCM6O`IExtlEMgXUy_IZ<IMhhrZBE~n%>S&&^L4C#?Fx5n)_MGABlj+An$
zFjBxGs^7C0lpC6!iM5PgJ5eBf?cw>X{5%hf))(|_j<N=oH*b7tZ=l>J>EQTk0dS;S
zTH^)#K&UimH7jwkD3>?w+u&eaB5bPe1MEi9dj9iY1a+zEnB?kku^9<8d+|JDe;+3)
z8J#03WLfgOC_1+|Mk;Kt;uS$IUz@5m<6i<ad~vuxdtaWNBZ_z(t?VkqAEue}5zXNO
zd1WRie~;=p=QqyvIkEhHMIJN(x}x~F5I6ui5c`YZU9FXe^z7ovJ)ZmG;_HH(U0OxO
z`2yVEIpl0adw*yb5yLO7*gktekxSn=0bST<)C8HaG-3~D#O>%UlWDc01<arz3dW7t
z12HVZ8FM0AXJ~n$d?Q`DOzUoXpcCQFi9~A=yPpvEgCgPuTzFJ44PD_jZm!}y2+HLd
zZqreuvV2OIzsEfyS}_0e%LRhLh$h;_2ZOlswQ-i%a}I%?T0byvB%>0Fb&PNe4Z-fE
zltkK%<}NBj1{s3WMN?vPy}~Jey*(1g>V2RjH$?4VS9X%1sd1LfE*I^F(PT6hD;)~J
zOhT(Fdgl%z{X54;aNs`zU@j*km3ZG*AY8z7Cz7Rl4WVWMdM%vMV_1lD>FFP9H{)3W
zE-XtGro5qO99E=Z35Zen>fJ>)EHpHR9VLXf`Z(Ib+N&QaIbV>Ofei{Fnxz__G@LsY
zyEPPij8eBNt=)w<*MOhWXeAfx-{p)Mau~{4LSjuR$W<c-oOF)bqWq^!xWLxZu~QET
zVF<x12I4Le{;n1Y1iFET4+u0rCJ3BN*ZEBq1SWGG4-P!>$r_|9ff_5FGfI?p5lkvd
z)yCUtlU_)=*o=;jSgdE44iAkcM|<k?F9`M5@wsi*Qe4TU`--G2FbVy^Zf78F6q*Ns
zBL4kN0^KU8i^ZPUs~-q~$6Eun!~vq*j`&|uriB7rH4Y^d8s%p~oCl4M(DlUAC8T!H
zamPmoNR@j)9ewwUqMb2|eZ=rz0;uc|RxAJ3Gu=$GuCpLD#43YeluJS|7Tbs-b%S6n
zZ|Iq<QIMspT6P?UHW%xJB-vl&lt6RybzCmMoERRT&wfW|2oDQ%Evm3b=RvOs;)}8(
zX~k>hTZG=!JB&qd33RK#JpAudNaYPjKoJmK3KAnCTyZw0^W)U~Be4-G@lN^UAhT9{
zpiW|QcY)pMP7>k~2ktt+-7UhXd<U^yzt`ZI%HmNjCE!m)QVpziwCB?>agU(f6^BU_
zdmQf9J7;pNp=Xzl4Mro*6K%oN@_@NDO<QjYHg~v3hvS82tT_hcqK*$U6!ypNCBh{-
zS}=b}h&f&O2*j-d%`+TZlMy=1KDG&(TZz*R97S6I{v`1olLFxlLH;B<d+FPDp8%ou
z+L9v`<073kqNA>;nS9Mj&@5_ooekCmxRNXjWPP8mP(4Yovrld>G_MdOpf;>k;>v9(
z*CquZ;~1nKn+c??IV9?gt3<f5h_Bz>(t_~J1z=OG(_N9(xIhruVi-xGM~;A^HD__@
zT3yfnL|Ppsd(`;?ox^1IRr{%+sohtvwg>it+~!Dd7of?iC=GQ9%LQ=2+SpymT+{AG
zbpSy=aY<xSg*>f9dWjwevg%R11+%dV*R8T^XObY((kt&M(s?#|II|$4i4a(W0WD6*
zBHRe<(6FH%6F{}+A;iy=?`m@3*L{xWSFxx)iiJ~Ml=B|q5_p#0>m37<8P!i`O*^Cq
z1ZI(8XQ(pocq|ZK>73OOV|sQjIt#!U?>HeIDaLynags=j54Y|zpyD)vyNFP_$kg%H
z@5@vuyi)(5mvtZKR7B7omnB=j=v<Z_BA{Y__Z>l+H|Un}jtGBx0lUc6j)!!8#t3{^
zPkXTqPbff!Km}Fc1Y}&Gdn}!&5+{i^W3}HLj$aALLRcfxgVw$KpIBf%31V*jt}s|U
zf-z++t`=qHNR_|>?bY%fR<Au=%`+SieU()1IXNCmQU?4>FkGJCB&z$QA|r)g;GC*Y
zh}lX2DKbx2ubf<>U@S$_bequ3Hr?rhxmSYVRPQ+EGh=Z|nSW&*w+8}q3gPQ)%Hljd
z7k!=h0*k%|KrKgE<27Qbg~iW|kld^|ARtNMtawJxv=5en6S3SW5NxQ&4v&xN(M{D|
z8I0>hsu>laa{aGExNazCZ4FS3YlY@+h*v&#c}-((w{E?;NNI9O(ZUVI%K}_=jGzgP
zQ;AmuxlwR4N&%El1+kp)J*G!|c;edvoeZ6XG)OC)R^){OFS)mVM3BGF_N*NXMNgHH
zs5%-cuMtMo*!0bf+ePH5pfQTM&sf^5-I4Q2vD}zV9uQ5j^Q9)T6Wi?@KzSHuQDLQf
zY<)T?Eus6K$Kn=I{`?@3tdNNwoW}}`5t##JmIX^-+7e8tm<e-(j9^{$f?k8H<6*$i
z@wP~FquECBJ3rnNLbW(ztHc-1fS|=^%;<_4XZjgsaO|vS7f1&%$s_Y@Va{sE%b1r-
zM8{&-0xWwLY20i$JE40(!!F46uWGyOyG36X>IUzTGB-NUE|L*xB83NmF>}8Ss`7Qr
zjCp#7W%kK^;2bD_37eq0?gMyRki~(WM?X@77tHPFI)~1gbEEe>o?R<VSGrm@`h>xb
z;Q@kMoGKy@A8ZQ;Vz~<dGW*yHQ=;{R<gu2(|7jH+T>`VTqR1q^M(ixeS#btUxPb98
zD;6U<<ys`fLWs>>C2kU}?l-+7Zqc)g%A%fCGCI7Umzd5z*tS>WDPdO7<WV{+Upv23
z{E&>5H8!Kz<DOLgK*1D{F}3T5E(FMJmY2ON2y)>Sy}lO5Es=))BDp(D5Y55JfqHVM
z??O@b>C>mh{ELbV6+BDV*#Ak8g@9cY8<@0^s(ugDm87u-I4EsL!pvVS+X!jE_I^+a
z-3&6_V#XLQ5nk<qo5z%ybyH;SSs9KeODx^I$$?iK`hD`sQi5owXiM=Gf$)Is8Cws+
z;M8KiAnHf(Ti#kE${Zng)<2y;D6&bu+bK#;mq4c)(V2FfCIGg~B$&c`de%^heRp(T
z3gN0t$B~Zlb`&<%5OS;I^j#2zpw{$}t3_otpu<Hcc$W|lOT;Q-lOmh4;drCWsgHx&
zZf2tGFC!&i9~ti-j?aovwzOR%z9q`pG9St??sr0R`=dQ4{7;#L&>-4ub^VYeW+O3y
zwDBZ-&agnL+u79-l^^l!+V~8n8H-AYF^+fey%l6W5NgR$PS(2|jM6!{)cT^FkL>1T
z>~p3dOA{05D5Jin<Cla|!v)gL{9S}fGOIJ*%FhEm<FUaN;O=u+q_?6bI=7fYY{z8+
zoHv@H<XB0sU-3#%vk0jq)@-aPBzI69Zm3JFEzFtsv=N%EL_703H)*#GP^wwnaWG3i
zGEk!|Wf2g2iu89YRhDoxFW+qo^;br8<+GHLe-q}f^jEmcZLO<{H#uaWAU!R*`N2ZX
zHtk2WO}tK!n~cFjOAp%dJ%KQr<psOJk*lFBPnol6GjPA)T#;l*uvma|Z&oI8Hy(~<
ze_W(#s;u!h0Zv22vUCVH{h9)j)uwjG*O!pCbdtV@1-a7Wto>NB)#Dvuxd8^oW$*Fs
z5-+mbmzJ^fC(d5wA^eZ({A)qZOOX)M=DF)yFwdC7-Mw+CNVBCE<$%WTdVGHoVD4B9
zFkEq&6zRVd3?opAvht>hm$?qW`D64zgiprggk_oP!Ad$-zrOf_cYbQxrv<vf2kI%n
zfexTMZvgdPhV`C2Y8IpxNO<lA*m=v2eU_3sgJcxON}BfEr9_)3!mtUJ+v@H@U8E)s
znxpYG0q}r@#RB5i{pF{SW-q!SnSp%pMi31V>GGXvH$i1-VmM^$9S8gj$TJU9NxI{n
zDTZD!s2|Kaf@Ibe_n(NhhFvqXFuKgqC$K4bZXvXq=-tL*NLg4_rDo*T7>1W5{3q%c
zIyr|qJ@K#zN~OCfP)KTCcq0Ekh~GhU-OBIg5Th2ubiZ_)dk7^9BIk9*10u|rzOe?{
zsy%K4po?~Obwz)Et`B0Rt7j)fA3V?(j|wnLuyJK^UyZNa?sw<S=!_rfIsZ&qb;xtc
z&kKmidCAK5CP9|H!^agnTQcyNKpH~^@fZZrnb-UUh(F%ukfs%r0?ahsz>_-QPlaj7
zn=v=u*0Xwb|2U@4^<SD|u}M6*S48nHhNMy_rBA;94ls9#v34zP5a}k=N@L3H=ZRkd
z=P`=@tJ|dM!GF0E)HS7|oK5z`>UR|hX(;v@d}lZ|7ix(?4Udfy>)f;?IYmF@PJ{tY
zU9EJEFy~5^EkTO~<t7+Yu*|;;pe0kxie>IDGQq1@<^lb&g)qON>)7VU^j`xwS4MaA
z#G}zE$dyVLn_$kyvhjC9VIc9&8JxGc2TDTGF)b$bY*w`<xx6j^)L%I~el(Sg;r#(U
zyRzBMATP}BzLy_d*>2g2%Zq4(-xQbzAVtuLxx(^jjlu^F550S#K#GO0DF(5B73JDU
zv5t7h5x)hRYB?+q6pj(^*RusXGaXM8E|ae=66lg(hlO&TLFL2u6$xc~Nej`VrwR0T
zIJQqmPKW;v$ebk!PM&jrU!=iaG<jOY{XjI&q?CKBi*N~;r4;Fg66{xkC>M0!6`GQR
z1iDygwdf~wiNl1^0P{L$#^?V~<dlxLQw-ukfu0oQdl~15o)VvSvcNm9OtL_>B|Q(2
zBscuH4{7;&pU^zJqI8U(il(9j?_pAb8Qp&bP*G;ii47j~kFsBZgSdEQ0IdNUV)%KX
z0HvNWE1uUg{aOQPno!~+e*%F+(h$bxBKWdm0qO+kP|;IuHi8hTAukYU=HSJ^^)k%7
z*9mi5$lIzNca(|Pi`3Hr=&cVqcj@@$Lvhx_&=jnSS=3W{Mx;?XXbSo8BajFg>w4zd
zK2cNSJbF|GO-W<MlZ*MmEKWu)EX<aGMz}?YD9!PdQduB%nG6D%UjFkEFE=;<Vej~!
zpkjFFMfRB1b#OZL7(Zl2sNbXRug4n#)j@TqRIKwj6pUoruf%;KvMexs#j;O8Q`H$n
zvpHPP=3kB`TZ^ZJIsK%3=JtFN+Wh3iYNQ<p381F9nxO+`nyJS^L3~h`lEx__JbUJ7
z=<f>2-4KHi3{cXd?5L-l5sg5%#J&&zy+0RFWrV)Z;J&^f*O&9xiCWrjeZEX5+0ojj
zuY~d)0`dQ-Iu9sIi{kA|kf0#3Kr+HENRr)|*+dW}=LIAR0{70`$uoDDJ7Mnv5=1}|
zR0Jdk0VN2EC@MKi6p##mat;zCDe?O~{q(%=e&@`Y^VV}aR;TLf>gt;X<r%siGk0H;
zcvz?_t+UQFYri54{xP08p*r(fDA%C7k}Qe;B_y|7vr$+QE%6+%GmzIQ8UHd{?kN-w
zw@sZM%RJ9F+*Ie$csXCR6VgT$o7bdBd{3~MEY~z$vFr-~a1`BfYiufl9_bg`Vk?oZ
zfSf_4nX2<eFmp-jO<P?i&;fa{QxED5M8e!pn>`&&KHY{K7pVZrKnj;GC0Mi>gT_5|
zP_UC6(GwHWARZ#j?2@RCp~SI5oC(u#y2Jg9Ae3gk{jt(Zd{P}t_VG=T?r>FICYvI<
zC=SOa+Na+u(43YeND-wedP1mk=4HhO8|EiVge^33MNEE~FfK7Z`+O_x6k#qOHpR*A
z_W4)mNoq_7jO)Ax;x=Ip$2NI00cMGvdNb4aH{vqEUIMWni<Mr7q~*9Q-xE8Ea9vQi
z*BQW!$g@h28>JW*N^W>gpo@uk#CDzO?NwnG64_JgsEM5&%y>_@t6i-$F=&nT-T+Y#
zK~2Lx^21CvYU#$yM{gG4@h8gb$??HkAkIzB!_r3JBViMK$IV@mw7>Q?Ahp2;vl6pJ
zkV`M3A?wwvBArO3Q!i(L?fi~|Fs3JAE4BH%1=&}x#uXx6a84!kaMpb96=~#qV7SE;
z=7MNn)UNZ3_;wjkM_s&SH4=LXb2D~ul&PbqPC?EAPtP5kN{+q{;%3x#xg*y90Kmn@
z=et|~G-344Nv$#ZVG$m8nRK$-1u&f3M=P=6|M<?sx6+T<DQ(B7{<&bL4@^`rrn^lP
z5jL9E8iODEm%jAq_zy+pu8xo#zY*=^>T{4k<(&`p(S?FtLOdzZ$tcAhpMbfpxUfuy
z*#m;8Ytp!&C?APmWkfToF0Au%O&QJ2jP|Jv)IQ3sis4x5Qxc;1EXC{-wh=`ub6pBQ
z88I%>oiI<`h#&|I{}Qfs1XloG{fuwdgVm@<8es`ETM4G~ATw8Ddl9gBtSe5^r<(*P
z1IQ;Y3drLT_a*6$L>UH64ztW+pBG^mG2J~F1?>0*kQ<vvsI})<>PtJNnxuEEV;xcC
z!%sY{*)bf`1yG>mKxUQ*b6NG~Y7>_Ygt_ZsAm8CzzUR#c=xi`6;Cwe;6hhsZa$>#t
zi(e7VwPWuV0y|mZ+4$cg3U;cd97{eXlBm6v@%TjF6bE(<#-a;Am<M@f<r<g6PQj<l
zkiO|Ub<SU~_Kd4DE&oKCKVB2<+OQ(xsDp#G^%n$lZIbWqv;rm2`E>K1iE2GhxMXvg
zO!hZWWQWtsNV>|ijUcK*_o{L=@oJG2Uzg0{ZV_e@uOzY0iHFMs29l8zFN~DXTzZjI
z28(^U$R>qCO58_~M@eIF6zfLOPP&(BAy|$Hfy0$Ka#w!cLQooSQ)WiX!v1dMd&Nch
zcJ<jeBO4-O#Nh&6E!45NQyY#W1WgP#$upP~993c%0(RrEhi8O^x!a;qMKt_-nHG``
z>oDF?1UXNeUut^z@>j|{DTb~pE-%P+?9&>&nP~Nyw3X`@<r4YMHH~?LFz22w!NfxX
z+#0j60@?bjka=~|WWylUVdKMs#T-0<FR&QjI=JeMujvz>vaPAN(}VEuC1gU^;>~uF
zP1j!>Tw?}v)ObP!#YVDBx9d0inv<bNXpo_D*jxyulCMAn-h(Dc^se-YheWz!h#qL!
zm|zm-f~zb^xk1NxLP5RaV;q5C!Je!ZFA8;EXds=!@wJu!$^BGTgYbKOK@=H_4t!IK
zaF&R__6B>TFt^eW=a<qhcEmjb;Z*C?nX%TAd{YZO4aq9CyGZlCgNdRUw+P5x6uB}!
z6753h8ioq7$WlNgH#o{ecKUXy*~IB6WpHT_G9Y)Us8?79k`iOv#A{+AGLs}XN}YSE
zFqasU2W$nD#ahb(x>)RZF$ix6a7>*wpgn5FnI+a3tX0xs-oimRvO~I@8Y_w*6;6l|
zkQslQ3vg+xwn^HmL|CP&tSc7@QGcxu<Kk`2<)GYzqk}^na!Wth5i<lkA#^gj0C}Jg
zqS0UG48@oza}o10+`*RNIXmW(PAs}SAz)$~zD;h^w>!iDf?VC2;;(-l)ZJFo8QsM0
z{|;RN+^p!5Z%56MKNRLx;>0Umc{@Rni=fFqz7XvqbE>JMjr&(DA~m}4G={<CTY_8?
z9-o5ENg22BN(IWHjI2`BMn?;E8;#(LwId!8;6h;9n4D)^w=$43Wf_wg^}Z15J;oC^
zU90d-V-2u_h+YxS8aZ8eqGTm{tWZw{oM^<SB3&t-HgCq~qH{Z7EKNt*dsUFk9UiAn
z4igV7F-+j`9L?Z*q{MU`D%l7<D$I>7RVS59PVOEPJTcqKghE2Xy8`pz#G+XYS$H)d
zsy2#vT+z3ggZpflT@&DP$$<rSo?NpRM&)Eeg>A{ZB6An&j>hUxnVZ^J(tWQNWENp<
zoVKXQ(*F?Z;xX&Uv<NmoCvalUY{%24v^gE|reK$X=qTk_B=%erj8@R;YBWSR*P)(%
zxkHSr1)0Z<n)+6>PAXFA!1^u;igyX4Wpqf3)nYs%npFF=i{4@~MCKq@x2#~<ULfq+
zh(n|W)^cdh;M?LinW$>{cw=pc8sf~m_D#NBdQ7glE(j0TUB@x!%5Pbl2r~%NNi<Dq
zGC4u0qse%(Uq0j>66SiM1!tNZi5E+V=P26cNa`IyRH6&bbbOqNezK0G*5#KxIJxtf
zp0QFNSXy}Q7O<Ry*Vs%D^&Dsp$GQ5ZTi}@8=i`&s1IVMYCoOB2tY09y7r~1CmXL{o
zHpor;#Wnz-qsVVDS6D^4dhMe_7VJ8|1xC+;3vz;Wx=5FbS&kdywRo+B+OcSqp!236
z3fjWW|8u7B?SaMwJRIkVb_$%!bKKC3+XPJv*lym9@uE5v6qaEMHXKzEq{yY^xI^Er
z1<xeV?>-aYP9aYfx;avd)mt1LrTxsfS%eFLVc48lrLCaZwbf%Y(QX8thz<8tb>pQg
zb`)+66NeIG>id@L(+?gM9G_uy<)g<0yC&9q4R&BaolsYKkS5n*V1jZ#)`dG##gJXO
zIUKWNl0@4}XbAT%tXT0wL2e}}mn5FX(Zb9UroiseI7fifXRDC)Z~s>&R_T|e7%vo;
z5FQhxVCQ)#qPinhYxp=Bj-980&=T!3-Tz)Dc!|3wP7*=0w9T3v@8=(|XKTh1)A_*+
zAF7P?$9bZiDEkuWtge^=;>x1pk@7A+669J?FZMiF%q+qnr=ig9h))EO<1mIP^;yNw
zjoL^u-n~SSnL|S+!>hQw42CfeFQX5~A4&*6Hq0`$xJ8gT#w5zDI>LtR<ubVjJK3ay
zUUMS?dLe1=?2mmnhE!iiU;F#~jXoy6(06Vo_DN~3Sbh^QDww7=&UfQ%0eMEmmVx@M
zx+w^Z(@VqZfp?0|A=ojH@b2FRA;XQPOpnX;Z6<OZ7_)1(6oQ<u9zNkfs2NLbRwPpA
znM$0wd|in9Z-v_lJWYdR#P(QKuqxBm7HfWoZ{`LDHJo@v6u->E@7UA&h69W}ro~wK
zyO3_S-sFO4Z2>NojN!TwYNnKs+>vk%Ikpf)O|e43QP)6xCYn?`Fb}5Ut85OL`KS3=
zv;PLdC{j({aN+<_L>TVo@OU8ZD<fbWgLa5)f3bxl;OP`MlRItc2uM~<Ok{Tx=^SvU
z#0fC|ZpVdr0JAXYVg)19Xf6vL*^{H)6w(;y2^QuKQEXQnZ-Qyhc|r-XJm92&6;%+d
zoQooOsjc`n?>Jk+zIiVJ6Qd6$4zK+R(bB<s7&Q^@rB;_}+}u&dLD#PB_2WWvztH0y
zaBIfaK%U}oA;P3FE>{dA(G>2HDsQ&&3(>g%NCE6mZYV?aV(b_*w<$uj%a<BsK#0?3
zMn@`@-x9jh<HB89w!4Ep>`a+oww)(BTh)4OxLpxUOdn2;YMw4ZIrGARj}6{S0w>0c
zjsoKiAzH!o_*?vMp5Hl6p=hgaU&N@eahI~DMA#&1|F`H4gmELFaO0NSe+8)PC)2}I
zcPxHRi$EN_6M(tfjgD11wFPztqLJo~;-X`=2zL!ij#h<(#cgMUxnrrr=Tp?}gyxi(
z--hV`^FX+o8`PA$r$Xh9id(m2nR}=(ml<^?$7LUs0OJyE;EKBt!n|TaQ?GbJh?_&=
zsM@J7o)+lotbtr{@UA8+df}-%ev?U@=ILV9BH!cJ%qh-^=EhS3TmfGM8IE_#FgT$@
z!dYi`!oVB0RrNSSgzJJ%PCl}maiuUP!chaW!<=~`{4OTL#J?+)=DlgtW0yVnb~*Go
zPd#=OKpt}X9mk7MNG|iQv!_GmTSDxPGr%N(9*$F_Hv|y6bxK<-x0gfnBXbm^<1Yf-
zY;ut?XzzcY2qib9+QC?AZ-|Md4gu#|qWQ7fL<%`rgbPZ4<ml~?GIU>^X8Um&fMdv>
zYP?ti)P?d5?Ng+W>#|xrD%w*wFQMY^R$g^0vD){6-Q38B@&vPi5DFr#Lmad(gxlU8
zj5WuO37lvQT{>lUI!PEJPdm0XvKYi{H3>a&vVLGtp!3AM`K<k*+-5j#O{Yu0738`i
zZE`35F9Mu=9|jPZ6yZ5Sn2U)}Dy8SfKPV8r49dp&rTsx@krv)=`_=(`QwgkK`(g_b
z6afP``sSD@lET>&Qnu=If)E!<2c~t>N{}0{+QU^Rc3sR07b~Dq9pzAbDYUN73DWvJ
zi6Xh`jWR(FRM^-K#|;M(m0C)w!Y&ch{EC6}$g&bYBm>0}%$ij~h;!Ox$4#Q#Us$I`
z;ttW~NPj&&+xV^!HC#LIpDxx8k&CLsFkG*a=}tmXQ}vHWL!X+gFwAMj?Hy3&IcMEG
z_R7xZO=0DMEb=B!AH=UO^n>|@g$*YGxF@ZZ5F<b@giIC048ihHW60ks(mCi97c<Fb
zY}5%xiP38J#~ED?4;x3&;I7&Y>BT`0ad7Nr3xkV<6NepTJ#G+@ntVp|_44iFvO!L!
zZEUQ_<skjwJ}hM={f;#zf#wfhBCrUcFVyA@+Qp6>?H!tuam6Q-HcU=W&^Ni)^uqlY
zBAkX!=hQ{WZppqPJ6mc=6j)6l@#e}G(~cq>4=;K+_DUjKzfky98N`gB8C&%`1`g#1
zVh<6H!M3u!4?7^X<9$M1F-ft=TkMVR6-c|1D3)sH4??+@P(C><drF9hTc6(5j6LfH
z)KfKd<9Y#37CDUBPP4*G0&@xQO_7cX2IFO+?g>aj$;ZPxf^1TS$?Lqg|CS!&7xw_p
zpKCgliB*NstNVCWwWob<98iYg*%$5<)3?qLHZk^je-+PGCDM`H(Z%8i5r#JELr3#Z
zeS2x*$TYScb{6z2rirn*py8iq`FW#%iS)Q?uT@9m!Vzd%iu*mh!Y~Svi(K#P!``|c
z`wBBB@G8K0N4ku$pHOoGC#GD!<hd0=E_a9Yqu6$pfx1VrIc6+fEXdj8ssbl~qTM`6
z`4uzA09*t*nGadt7v{N~(-|z{xlhGhzQlMD)CX7iG~Y;=(`(4RT3jc<r9uJDs~PdS
z5Ngb<Qi-kR6ltKVpneF79l~I4E2e6<>D%csYS?VO*#vNJX$;^)FKIj;oC}x-uf#G&
zd%_<T?#jpul(rf1xG+~nUL8?&BvYlA1v?{N-C)$bDZm9sEZ6D8Mh6!``gOZH&Jp0`
zEvC|YUOc44*j3P%J7Tj#!Kg9&Vy<L}AS0Y`THCn94*}ei*fkE|g*Wy*42VHBWBqBk
z>^M-A10=&Bq^61>HzO~tV4rh1v<o{_>E$|d>?*{Obj?-T;P1~9hy$#lM*KjCE8Wh)
z_)wfQPY|7>4aV9>7AZ?Z$u{jDLUNO!w{OOa0?Y=aA|@u9eW4!}fzs{hSXhADZElaQ
zEy~zs$r42J&QF5wE|BIQW&N(}8_sbW-i%F;f`BWXcz@(Brzq!IS956R^@ksWxza3^
z_^ja7aY4>EnQJo*^cl<?mB&5`{CpWTP1!6+8y-zm_Y_t%oxdl~23rfJOw!y(90Nfq
zxXq=PIXOD&kAsD|aaim)S3DM)3bo8=iEr!Mzj3fviJE8^kJVj@8_ImKKv#|PD&0}T
zc(M%0&D6Y;d$|N<kC^0!Sn(&$LIM;?Mu%w1J0x?plSH`jSl_w^W5J&S<e@XF*CJI3
z`s`}LiN#idGY%0hr52F!o(=??t&b~`s*YB1&$^ue7rR;StLsth*hv`4pw6#Emk1h=
zsj<B`-u)RAY-vNH-RgM0-G7rg^Ke`(gs|0g+`iTc4%>pCinaCauo&Iw!q=b>GSL<z
zru`hk^QCrUBeM4Di|vKxNt$Dvc9Eoxkv*coxuTpR)^JMkQXv|osGrjD*gXPVHx@2z
zmtfHoIg#W!JJ$S#qfj|~eX@@k65xU$9`L4C8W^_=H7l_##=U^{FA1bS>M{olj#lu+
zeo43qX$Ug`Sv+weq%T&`4^)<;BP@YM<at0G-le3GJNjcU(QaMNjhS$A>+W)55f5j1
z_(c%y>frzvHzycBjR_+o)Qt6bSA_eG^r<5;`&a(8Pp+g+I0+&*Uw68~cC`>1cO#5;
zTTgaSTtsk5Q3NH{{H&`i*NM&v)AvW>DFH4@r=FkI^Dq0J0_OA)5e5+;J4%G1S3~qT
zu|$xtBvAfJh?5>1mHzZP0dxY5C=_#_{2G!@pqDXtH~ds6vx+@Xy2z9C)B6ZFi-*w5
zMP0O0ttX4UHBJX`fi>YLkF0l;SQ3>m_K(%iAe<*1lqgMk*ScGfy9n3Ty5;_W9(+%U
zA|WNgAF(i@6rO8?oY}^M3S^dyCHEHbj1XAfisSaz^zEsOb7(4KS#iNLfnD|yZW(vP
zx&mA#?5&a4hU0EQj#|}y15Tf1$?~aS*Aiz2Y|2p&FLM@{S%Z^~{#Z`5Q%jwd+eYUK
za*jOUG1QDp1;F%lbvpiB{#+#+eBR}sr>@_|%D2YZ&HxDqok*>qkx8-D=b?K2PU`8t
zsm>lYI0wL`M4QQILY&yP#4v~BXo!Vkm-C8n(lFD@`eKg~OK*)xVREuCxICc@X^;1X
zXjI5^$;rQg@L+B<byq+qG?$(4I7|dw{lV5N{&WFI=97FRJyg;SrZ--y;&+;sB^c2v
zR=v<925poqR{w389*&w=tIiVQT2`?H#5%n%9u;Or^AsrC+Q$XBY(By0p}n6LXh)v&
zI_uVi^Kbp+4mFC|A!g`{{0_v;$?{8AXvA`Y@<IRuIx%po#eM=EbAVZYyf1DM;=<^i
z8t$221ONx{<ez528%5^;+ONf5F9v|E?A`|BKbM%mMx+vFTxx=&vgvWB2oD(9`f&`a
zW2*yy59(6M+<}|UETRd_V~a_hg;jRO33Z*g{!icx<=Ape#0$*09#WDMgBn3t`on}#
z#wO>Pahw8}ljuuTDb_m#emdc)`pNugj$+s_7?%s7fwB74!SS1!#M!O3P#a%Xghv=<
zd5P_X<Zdj79eQr?@4^_~Q)f<&`}9pMSp7&KNhKFt4&><zbCvQ4=tkjWO>dkt&>we-
zbXV-J^mWHeqMZ{Hg<UBu`G+EmPIF>s(V1r)I;jS$UQq;T;0`1<674E;uU1!j4;SJV
zuBIo~&lBLBdMhwq4z=GXqv7FHcPUo;Q<>0cKbwu%RtOVDFYVeM$BJ@Ubo+!Yb1ZeG
z18~%Xfb)VVm!xJ7iL?F;0^?dR;`^??`4uL!Xh_bvPYC?!!B;<v-p~JHQX&eo?0#24
zPSk>JbFBPV0CyJNc-Nb9jQHyXI!`<&v)POrg^;S=d1BZ+C6XeH^LXdi|Hij#;9Hpp
zwt}2!-(Y=?wRgJ+bfO$1ps$X-g_wimn9la8?;KG^<zg~#(qje`XOvM<E9fG|>VGeC
zK%B`3ZL3^efXU?_Z)Nw#K?2=DkXY(*a~Vjc@641;K%bW&f!e!M51f4sF*P&ln(*Ty
zU_cG)0|xOX*Fu_)tVEct?k9j&<@yTK!|S5V0D5$PGfCtNT?c4JbvL^(5voOk9FE<f
zs=+`{pxMGy*vk;gnBxv$`zY(8h0EmFMWsiZ7Aql}Ko7^#f;?;JfR%R-N3qK273#r-
zS}Gl&r#C517M|N^w4Vte-Vx%8HY?qAZjN6MK+{i~IxWuEw}W?}W}#i469OAJ<X{r)
zy`hMVD8QKi;6?yf4KY^hR3o;ysR*QvK`s7V0_lcITq}Tvs5Ch{e@cYgv#RaqiZ>T@
zXT7T{ekj^i(<GlBFw&vOuZ5Ecdj!<sB3y9#GkdN0gAlg_V@a$+lk=KDbDc4d$tf0q
zw*XQ7RvyG&^VZ_$sne_txls_g;EZ>W3s)vM3!f5u-Ui_;kzb@7IYS6ZwajRZSM}{?
zN~#AY9k{u`?Vx5Cw{UfxcO*6}G4@U98VR?VF!Q7jBegicq{q4T9nXq(BQhs3>c8?2
z04g?I(?f9Sux&d*ba0&$v##`uEZB^uP_&EUn_^U8-8;a{GYZW?g_hm*1UfJ5beO<S
z6X1%YBG4B0dLd-R)|CU9=(!UTp5cRQG(Nf;!ujM~a7TQ6515n00Vt<W-?`T@+VPnj
z-xc9snFv_E>jLuNV^Z#taZX$>(A`2-`r!7snsDcbt3!^qn8j~USYs7LXWe&-n}p@A
zQ0cC4S{hFaq8iNfc!ZCZe?r1B<TUPabB|JVE(UfA$mqPcd8$C-^s$d-wYp6tU7!uO
z4!7&uU4Y%W+FCQyl>3TQSU1y(JQM>0T^OxjTEPBVLJirkv-JE|5M!}|PN@>h+z;gj
z#rZu?zLhjym^$O&2l(A$GFAaey3kUv$AiEW809Dm`xq7JoOu~ej)CG!4;N5&IvN`Y
za+7mK3TY#@7eX5&^h}Et9)WOC<mx517d??^9X~$JIKB+RQ*3kMZP9KLG<K*(Q55g{
zD40vkkky%cnzsM_SOKFXlz5!_cY@8{F)sGA_^PSrMQdD><Q$cOo6wva3h$~KVzI}I
zLZWw&RyV#`Lg`US23C(C=b}qA$oNu=pDdILV<4TDn_oNutx06cj5t}}nlb0FH;UO$
zLYZ^4lI&6wV&>2RlE&{e1v3An6NzU^dTJ}XjJyA@NJQHtmX8CU2AP;%sJf16@KGbw
z4AjXHp8Gjf9W?AR$c}*N@<IhPKRD!+5qv#f7M6##EYSwz&}RyScj;vBahebpv|mRQ
z*NUbWjB?%**JXn8RpV_DZ~@_KcqrmIDEKp@bxL&R?~U*aQh6TS)fnedAM79`;v6f~
zB^hXP7mXD}h<T>XUcK5AEiVFD{lH;8x2q;DD(|5m9G1LMZ*t%v$kj#H!tkOQzIYjI
zVhTmU8k1fDU|=A&%kjdHNdJzylK01t%)hg!OMtpqBvsm|b!yx*@r^696JG_V;+zO@
z=Kp{w=PZeiX^KmNuYtMJ_?kcykS3CZy6u=PhOnGWy*$C@1-a^;^!<u1&a%*T#942c
zLiNXS@6)1EKb#)Rz3K0b=#y9I8=h!b=}?<4{1-tk8GYYo9#0A*X`Dv0Df~bbjosZu
z@Mdp2`z>fV;(O(E!rk6BfG8@v#DO?20|<G%K0%bz1-LCFzi7Gpq=fW<wx*<b#~G0h
zT6;W7il!yxtCn*7LZlO6ykXFT3*>pAZr+V!;NK<4jM70Z??<ihZh^?YmJYp&kSr7E
z-K}=D`=11o8@V6EBJlSz7DvKb0C)Wl0buGh41*umx4G7Ze@ESokJa7-V-Td%o(n|~
zc-G7*(f@uC825{|_gwUY0+UXlot;^%SYixs8%wM&%$)>-GM)KKet1QwE7?IzkjWo{
zxSMjEfr9)A0TX;cpWhMZe^dmK;=Mm^7GO5WG9LelB((W|i%_zsL}w|0gfJ3<QPR_n
z2yx{RGIb8$jQan;+&D-`+MLFs9|OS;y<Hc3i-5a{SCu$SB*pF2qpVtNE)n3epxNX?
zs&ujmbVf{1QZaSJoeF4%)VLnNA&R<6LOp7v6;96fsLX>3Oc8py85fUcqN$_C7eu5{
zUY8~3H$QPeO!J#@oCv!AIDLZyK2fe2GrtbJW5rJ$nXWQ8jF0@!9Jy^H=IEBsA>0tW
z_*|*R;Q~mF&WX~n@kQ}7Dh6hT(*@8`W?<WSjlSJxeHAV`4#tf_Ci*;X9H>)*%oL_D
zlmJLZyL@Tq38v`knyd^MPYHE$bW)y<kBb#9BP<|HiL>V`5DZeaXjc{lah9CN=uM_r
z|I0=4<LEJT_C@k93(ay+H61rfHWBFQSk~M4E*MOzT>sz}qO(zQj@~E2J2&*c=n!iT
z=|XZWz&6u}dCG}Dw&cq#L@u;za`io1L|#17qowIS%PvBRGr;}W<OOvf0WJX3Upkjo
z$2%e?)mvc?a-0aC(ZVsNFYZlPA$h-kaVA`!)jRWp^!l|H236Z~U%WeZ5#?2}r-I~l
zqi9#eTkv=zhv(8BOUKwE{7xUlE<nyQ7F^Vc(W};+8M8$>GOHJs>amv)l209+j&Pu8
zbA`2h90yDS+#m>b%;pk#o)%~Zb0U%s)ITpHO4r^W8+^qnr-_##|DI@fF(xPc%#zzT
zz6$2@v`m>I<DLVAxVm_=?T#ZwJNv?mQ~Xw-wNF)?63nZ8x<NnXl5wkmxjB|x49FQv
zL6AJfO%~>A_Vys4$NmCfIvW!lz8#Z)mN9gX6b*kDplPQ*)EU2Dyhsqw)0N~(@)2RK
ziChZM<^LzdB~Ozv3vqu;`x>CD$ILSpeWG2CdKWwE__+Ww!k}{??q34JtH<EbaO|`s
zBza)ugvo^(d*`J<$OG3~WXi5t{xfE05XQ6T<^HSA<rC-~4T=uNa<(i?xI1uKM0JeD
z;5<}mm}jU%?RWL}!Zf?FZZT3%UmDU?;fCsPTqv5VbjcD>-fmyY0Ojap?rz2x8DNU_
zHl%BrB8f(24(f8z&Xt~qIuS|wxMjgSM)2S@;dbUxp^`-qkWLSO_cDpTr*Fz=_x$m>
zNVv&HVrrbSoWsq;3F&A0&fTg$$Z9i8ffMNLb$FW8&tH@Qb;2o8xE6;k?_}9n^u#$L
z+=Lz2Yc}H&0l5}vY0|dm*sp_8-WIgD(^lZyT@<r)-3yQNN~nebq}I0X6^m5pP~LyU
zN={`ARm=MN_83SyfV4~PV*7%h%a!@bm4k;vNn2*n9TdkhYy{(v3ML-no%C<|cA3*v
zWpO@XE+=O!>2kvZ3g%wKG_0QDt1Zt6*XT(Pg?%B)5!;*Ta3Z!^1<1K`q{*f57+VcQ
z9GTV<_vzbhj3al{0QesmiNV#&IFjGF1pEyUcUJiaOUel8y4L`9elpHd*?R@KK9rKX
zX<&1NxeT~tl*Z{KA@0KRm=q6|389EfuM<8Y$VJOeo|M$vWkC7~Be_2oT9be-8&}*L
zynmT2z5glLWkcHNV*ZSG1i8b|49q=klK@-{{7Uu3Hzz~8Y0G0vOYG~c#ZRt-OdW9r
zdA5+;*z%c{jHs?!8_-$spd~xSe+bCy6kTL^NW#=`Ec?v@W)q)!o`li_=R%{Wkz~EV
zI)uy_qg7#HT0@8f4IwXxExQYHF4zcT_~k<Hwk~iQh1~zGi=eB>a&a(D73FrSRAoiL
zH1R)SPIv&X71EG?XFVWKgyWda{841C9t#Q95bFbwDB3EHtq$4%D&;qQM%*pJ4Jcm#
zIP{YglTdT5zn6P{GIKQO!mtU%1dg3r3g{|fMS>Gx+Lkijr&JAVy6Z>u5CfL}7O=Ub
zXCW~JJwk{pLyeP}{VxT%^i>{t<eg3d)J|74IS*T7!{YZEm*|G$$SDBsA1o1Fac@aa
z)j8qfQ~BMwqQTGs{?$U9D`EvQSw}qC0+hzvtY~lL+qLHv0v3|t+T8e239@8lt#$%k
zR=v5{AAeIQ=f&wmGVf^vaS$E=gg0AE1AzSl=#OIu5e}d3;Sa}N8KKW;;Gk@}PKZ;$
z*pjl|Grfq!nubwa>^u|1B}UMb(+n1=&1V66HsYCfP1m|>IS}K*-CKq@#W+T!<8peO
zMX#R<q>x-frvM^cNZd1W?K?inKO^(jW95yCpCzd>aZD3Je@jObahE7fq_d{QEBba_
zBp|XS6+mHZRvDj(ba#}q(!8{uzA>oVOYU7z-Ng4dDH5t;bIjAr0^B>%R3V2kDqr6e
zjGU*nOpU9*UHr~PQCo(6Tw-%;s3xS%O;~PxX|*U=Ef(C2$Od~^#DLR{6hQ1(yT)Q=
z{q8PU>tLw-Jk#tlHR;5kMZ4NqKyZ2Q9RVI7n02Myum;F7-vM^@Sr`)gK&J?G>P>vF
z##GU+s*b05W)QXCcfnve4ry`l_-9dhSaRsto_B03Zvjj(^tg`H=Fu&{;h10OZ`R|k
zz9n$xCG$(V_nENV25d3th_OQ%jEeT4cIc143(M@Ip?nYK8iDRWJPX6R6nB<UW!sf}
zdb}mfIq;GvVrwPdD<LT=b$qqjRz(I$hszZ|y1GCYm~bQAl1bZz!L_~~Mk}4NL$uof
z|7L6<&liyMV-nCry3N)F#8^RT(h*fbE;~;NsO;lHoKO{aSC~zeL40*2mfwaT44*z6
z55*HA9G*#!D!eDa8RM%UN%{wGTZ9=vBi<ipW`OyfO%j=`x}B56DG_fUig2Om>K#4o
zYlOIupdB2ImqoiL(XnKxq$%O`GSploub8qup(gBz3O83|Fjqgg8Iz3GBOiJR^n_Bp
z$E@`*;m(VF8pjgp+TT1$pbDXL$z)<45CfmVF6>;o;$US)*)^@|%x>8oip=SeH4bZ#
z0e>&hvsQbpJ#NV)+az5mUVcY@)%4Ac+1dJbXV7C~>ijVx%<Mq}ITDwNb`$X2Bj*FW
zBD)h1ZKlK1I9`Nv8{`E0mm<w?)X3VyUn#_y4DhVqfV5e+?p$C^UA5MT{`zb%3O0S#
zl-MJGw^s+_41JTEZji_0BFt^<<vUO(GRimEr2tu3uuw(!X>elRpi!&3%-l5W7C^FK
zFs9R^O_hSnD1#aqRmGMi#x@DtCbo0i33F^+h~+^=nZoWa)GT5Hg9;6$bP2KACGGxU
zf~W^h&MNWK@^|GXne@*kB>Qd5y5j~xx%2iUhxS9e7Fpq4P?kn<s-QgVuuY*n_m&89
zJiF?6MTj*Ata>`(!EJNe6}`P1zqq+4$w+p$??Icv!|7pqvEn*`6rB^rsj=Jc5X_%G
zhPhP~b)UF<&jp|P_H+Op&T()nDr>;gO|%Bb#!{H3A120~-U~?k#jhy1M~+^`!nq5>
z5f#{)!d*)?)~e-tLh|H-G6(M~0^AH}aXFzJj&=7c!k|@QB7R1IGu6#VoT+O+`GdVd
zQ~hSlh;|XV2V>Mq%RIUd2sQ5In1X)#eFsJw9*fB$T!=CDU_CKK0EIw-<TuQ4`zc5x
zPFIun<vUOMGLpf5mq(jRFuOjbQzy_}61#NNs5JGR3Yh1|^f3B=1-Q!N-970&#EbR=
zat-sLi7sN@DwG=HRG<<)KPZBrLrd?LP|LG~I<0p4wc7pv_IE6F#@s*<;mi^1Sk+HH
zpa2-^7>8aiAa_^P{!GO)D<G}}+f(g@JK{uP&POMLjQzN-gc@}_<akAp)55bM=QFWd
zwTQ=g7Mg?M*jSJQVU?>v^lc#%?SVokZY*QTyP&KfN7@PFLh&?-A0EE529~=$<3iS}
z-1X@!l82T2#dShR9<8tk_4^q>HKEC|9{Y3^iC5{l9Ef7rCCo#B=W<vq>v(R_a*%#-
zuGsppnU*7*c|h*VF^F)IX3)cQ)`s}DnWH2p_Rq-%$97q#2{%VNx$28R_LdNOUq4Q2
zKhd|j!>*Md&+9INGZ(T0iuHSpVC%-zh;}i~>~#VyjE#%*?F7<A^LRi2{feu9xJvDV
za3MQz!WNr|Hlt;|i4F<h+`^m!z7biD59%*~;YyXx@M{5{xI6ntnPnS#k~Y>I0Cv9~
zuE`5al7td!4sy*Jt(@kH^%OJ@oe9q{ZNl8^^fV|=eq!S?MMkSU!)eQ~qfiFBERE4U
z)}b_ZdAaWzeN)%|9yIhf4?#F%NrSfddQhO(!b!L~{LVp;ZVV1Obq=vzi44i-Mn`;K
z$V6w9-j@Y@Oqh8jbFO+EDj;Xp#9kR&sDZdlpoX+=R9#y}>10pSANLINtJ{L}jz)YS
z+8suwIJoz=RoBPDoiv*0I(I`TR07Q`HqNY9{jqF=aIht<R<W-r*fNg$k^S>`FXO#e
z-)^K%Zu_y~YJDCSYCh<$Jwvb|#JRF?Nbo1u1UgqKbtSB9J>q<t^{zNa-{uMyLL4e?
zISK#+5I^g2kO)}cX@5oykxnZ;heY!8j{%t@dQu_|5uJw$clEgbx$!s%2_gZG#xWxN
zvy{DfXTLy*3$3lXp0DnRwdMfkfhbLc?@-oaec^6b+y<lDp{w5|%!R@wG<J5i_?Mu}
z8VqmgSPu%Bn7<n%QpYls-Vo}%^+F)W9?TSbHjDH??230T5LI9>kH+gFa(-zmhT`jB
zAeu2XYrG<g&^$uNGbo2ZI17fCY(KG*T2~+q-O9U<d+R$VSCMn*j_4FLA^dg5wucsE
zwWl}s6KUSya+zVLA<HP(@Q2`eUPv!cv(%hZVltJYKqGNpiE)5a<AK~}TqMkC>XfjH
ze`zYcShx#k0}&JzWl%YsFOJ?nDwE}a4;dAacBR9J?#_j;bu>eh1UNlT!scRV9yLKZ
zYi^`6DN!Rt%1edAqcM5g)Hj`sRW-eQz1`tOGOXb!%sQf1ko)0)c4Uhl0qyyYIaq>F
zQVlO9m|!^SOTJ+*$h402D;R*kA=*8MiIx0f*B^nn4qYfd2jfWrn!ILBjjtWWw_6vM
z@rsh>)R7XjK0=HBKLzGw@YKLfydML&z_Lq-ABc9N_oD7OMx;Az=?)++Jlc_1XV~yI
z;tA0%uHRdXE01xAv=3*7={OeJCG<{R8tB&qI+~_g8fL?vfVg-p1NcQ=^ruCT>Tr6E
zWV(>tQIj$fx1%M*_~FiDJ$4r4a-$o<V+3+>d+Z_9$#X$|bSS=eTmdvk8>5|3F9Eh1
z>@?>886jYEFUP4n<?j}}KacC%W$%?AM9dV0F#4F)S*8{}-V}Wud$rv~nafkB<0y1I
z(^c#|wWytNf>T51$`OMWfXjtwY__$=)<5Ul%wy{0=u(F|YyF}CWy8<OH|ZqMgHL*+
z<RN-}1<Y9T4Fcyk5SE!(BRhDrwIElB%{+$*abk%up{e(rS|S{V>Y43Be@QG?OpZt5
zIMH;U!~~2aL^20W)A95zC;B(!BkCE?iF6s#9L!p>%Sk{k0}o-T{ZA3%K)M>CLM(W)
zlR+x$8|{moM7mA!`b&?VEr23*k2Z7<D-IN`7R1oBu5U6(R|F9hM&d&u9`|fG(*`~1
z#^*l;e4?8rzp0U+Jj$52u@ixlODh}|*6VR}f;B`^VfNd7(JF$Hpj?-D8rz9>^^j<h
zePWGYgE&c=UGHO(<hBB7Fdx{+5MnrfCCtS^v5@$1TM6-KsJyDg5vLN$t-%}Z_4rUU
zrKw6>KjkzC4+*&x=8mpppqGVe8sVL*SMzu4)LHSqzRgz7a<Q}e=@|g-M#!!75ays~
zh2?EH2R-sqE|Fj!n3m;{dBz@DJzu6UQtun>;xci(Aj}M~$zzS4vFlkyR%tEP!?uf@
z4d(I;SL)oOxk><x$56V;2)q9rNa|6ow#TXG^6g@wUFl$qrArbwo(F99H^(%qMndwy
zLb)X6P$O<pFtb|Lf12O!5JX<}Dz1Zu<HIrp4;1N6bbOpcw4-HXQ?Tl9h+t;vkV=Ls
zv6V144yS`1afE2+lP<^6el6z%xrDuS`Fe>}F91>Zk{V$+E)i)ipwDAne?kB)%=^Ce
zcuj<(V+$<x%m){OIM4nj<=W)8(2{>8c*L2aNL&*g)mrIykmd%K!Q;BNf<3N4*97+*
zv?TG)7v_==(G-$aZW82zjbR?$A3I)D#KRSQ9~bC4Vq6%x%XS2R8W%&kv<-}~(2CuW
z5jhG-?~cW@LR=&{SYXe?1hDWWfSCzc)nm>mz~$+a&l<YRUV_SXR~OPHbc_lnPg#+$
z!qX||zZ3)pavF#JRD}0(NI>$QozJE&$)Pa2Z^WU$cN7Gxfw)72IXrbb2Uj0vn#M^-
zj4NHn?=CFXaSV=cWPtR$>m68_%oIQ?FrSTfcSgTR4^xcD*+iwq{8*u`B1}NQuf$IT
zW!|tPSb6+AfmEkUs(^b$n8S=i`f@x{CPPOa6F1)%<Ob&XUoDe;vB>3M&JC_5v-=eU
z(RMmB-~l92PI;)&WKoLEg)si`9};`&+jUNFipJ3bsA89Pk~n-837eA>dHlCy`r}JM
zm*@xQtMe$PnfS9H_)BrqqbXmAcCFc5utaF9-S~q8vG>-q74fiW^AI&>dax16bl*P~
zQMF6bwVlDJ3xr{8t8g_T!Uad2LGQRyfD;<SflEHtOQ?(4Ta$*qEAGF7faW5aXkL>+
zfBU2`wS{aozx5{wSb$nctM|V$iJ7C`1A0j0#6-${1szc_`q#M<*p<}*3}zZA^y>mS
zZ&|M;PSvLi!i=uHVf?F%!fT_L@~d>umEl-+SlowWn?DoP1(woK&ydCUgt_{703zOx
z1W;%xk9i8=((-pk6Ye?h65#yxz_=E#@B9Ubeqy`CQBkf}x@^GSR#Ngqf}K%KRvxkL
zRgNW3r(>~+sN8S8*6k&PjJi0mzzGCK7jcRpctgcid_Ny@xj-k3I5#{L2_T|8w##ap
z%<oYcO4uchcl_Y5Q04=Vv3E0Uc@Ok&MJfa69+_bN?kISnp;LTMRE{ENk~#q1yTlM9
za8JORJrBnH8lK`{LQa@DhHG8vKccCqE;B`9WlMm=wB#!wS6%G_z&+;HEv|v0S#$%4
zro2m}ryyP@>xmn#g-&w-dzSsKgK%O=CrmFtSrEU=Z+YA-0&Zac#vPtX*F(81gNS^U
zI9PytAlnxiT`-GoaYF%u$m9*SCeTeifbfI)9H&<(+RLz<G4sqHD^F}C(1qfKT%Ocy
zL{*TPN;x(5yW){D*dPN+Zv?D;Bf%(cC!2wh__ipgQ^8zCuSXpq2;OLu$zZujq|46+
zzN208M7%DLxvI6LHCDWdZ)$9_ChBps0CA5R#M2@&_jFT&-<P==hzt-{Wbx1^Qu5RE
z>D>46cayTuxrJ}nM{k&7@!Szt3Uj~K)Q&{B(5)cmw?wjG9!~;rTlS2$<K#lL12iO`
z#4!R~BFWfd)6;?|!WgzG>)&36M+e>?ZwjEeOkF$-9sd<+e#n`Qr<qy*C?cg<E)Eyq
zf@(EIbGXeN1%h%(;^e+UTvEi(k=XLiB1AtgNy5CBO9ZpMmMN|AW{IHDZg2s&>n>uM
zP3b-{lXa|qH<%|~{EXvBf%OG*4WYC=!UJ;8H|{BNlM6gL`jP(x$$deGi44kf1(|u=
z45P75xevf)?qlN>qxVBQcaA7g1c5F6K#>kRoOEAi)e>SFW^DJz3_(u1(qBcfd5!>g
ze;J@n+ywYn8Loen#|h)%66nB0w<n$zke5;<Rran2{R@OrV!lrc_r+tP>5)CWJr*D2
z9|w8Lf%z=ndeA>s^|0gt4?(y-a_5I*-exTFFqrv*8?E#}_3?s8kduH~eEAUw4Tjdq
zF`{o*B5gi+s7rJD5|0*1pk82!?TB3k<&H~EmFO1WtmXJG*?b%-%vIweh+aKCqC_g(
z(?V|lX_*kly-aQw2yhS8B?PqTbn&Gg19T;N>$nJ`|E>FYkzs>}JF%qdh`WVpFk;77
zdxCHJOGlmSCh>(R*NiDr+OKpr^pz(ap>@ivSXtjhK-bRxdp}VV<4m(gHTp}4$wgf(
zZp`spFd^Fc|D4t|p8Gsi-=sCH7kA>Hq8NjnY?_$r-WTZ_R{H14evFhp73cyMiWZLB
z66|7dnKL~LnjpGIIwpoUpO?w$_<>~(@C$*Chm8l@;^A2EsWMx=Z(Eg#?!p2cuDdqG
zj6E1jln~N>8UQy6%JTu$W@vpR#LbP1B*b|}*CS7Zd313@4u=%a6lv3I<ok;IIYmP3
zIOLXX*=Gwx2JS4cbA-SxJ(f<fXFUg*`-9XtX(qi_pu3;^ShUAqpLa5JTNVbLp1&`|
zvj|fw&T!IM@CQP@-RB7xoT}<t1FFzZg<JBM4vwvCcB7hpa58L|b!5rykLY@V;AWc~
z-^o1b5J3#8D%PK|$cqqWMYlXNt|XfB7KaJrF9DdNX+$;SJR!vI(`&%jzg+yxp|b{D
z;}sC6IIf2^c6=4u6_GnIu2fr9`i^k3OS+`NntI&30$p64iE5AjnIPB8Z($_`*($Gr
zx>=cf2+54|7h&c#BNki2C0{R6V=Q6Y$<0nd6uvT-edKi_==|8h^3(U<@K5M0`r^kT
za(BzF2A6yjh>CGqr`r>;n`pA*Pze&o{xAn%f7D&+#`ak>{R6s-_sjkx(iLH)8gAkO
zoI?NmEl{^7F8b%uyrv&qo8J1|T4%hakREvGX?PqD*D?&Cw*{LM1C?Ga1nu!3VJ`gu
z*QiDt?eX6-m@Y%5r$iQiyNHTWla`Q53<=94i1AJdf0jda;Iv&AnWd^aTbMH#uZ$!m
z{BO&M@`lYB!j10~Nug@c;{2EpxZkCF1pj&01oJ>WrvC?m39%h_pkw+b6dv(wvBG;f
ziZ&ygh%gHqxIkn6(c#BuLd~f@67Gwo-v^+j^u`_&VEkCL18`)+)rflqI2}3ZNirKg
z!i2g<GT^weBliAVplucqu<;x_R*_d`WbJqKi+O;7IE=5y(jOG*cB8XqmQxa|2+dOv
zUaK^mqg5F79pFL6ZS?ICVk_kIvJwvr2zLfuJylfccq0*%=NR+?$uc*t6zJ;VQ~_CK
z)eiw&y+T8QvPx*?f}X&My#>%ewR7fh{y33tdAa?7-9Im5aEl0`>|OzsdivBE@qxZ`
zTQmo`jGmT^$^Qd&CJk9s$8sM7xDw<eXNA!yAzqngUTem`1(`)UK24S?@0P)&NtQgn
z#3uxE5!=ySW7jcN2qkL64gbUCueyau&zS$yB6Hb|s!kn(oJ10lFhneoA><mgXs4El
zwwF4O{;Ws{Rc+FjY%7RfI#<SC*NdPSt-R*)SpIIAIWx}w0-T<RnQIz1pNh^EU^PG;
zG5?olPcV#!=Q+q+NdW1hqpZX?L}b43C|p`E-WBF~fa7trZ5p_>NN}D9+A$zQ31H03
zA7K)}Ii#Vyd{4Z+N5~IzF{tifrN&IW*nH%|+{om0P<u=gMOT3xy0m`L{E+U3?6)yQ
zl<;u5Kxd}Smed<P3jk3UEOw?xbwLPsjVjmK@G~U<=1iYr_W@gf*)+RSw#1mPWtt*z
zsy-6GS;(pJTFPj=D#G(7hx%OYK~tI9<s<!&hX|t|_Y-u41iG{8)vBZ~>o@Aw=&v{B
zt$WdhiEJjK$K()<xNX8*uVjh@v~w9u7jTlY?@VE^NS8Yaa=l1;#4NlYthXrNc~H<|
zSUd}Wxpl@!B_91s@iRO7dL@qkDuAni+E&_s=JYET19V|I!<5O@4MNoEa8CT9zFk4t
zji8~|#f~p64(jT(+ht@U<`d?0s`V~T1LJ@)RMi{WMqFKnlGwr)_IV-XHgn44*yd||
zXCC8HUN%NFwm>J`RmY;9TSh|6IcBe9UN_$oK&eMen-vR-nBX?cIFH!J8NzZkaehz6
zHwvMfXjvT{?vF`J79p63wN1WFi1SIdeH7--rGQ*Qj_I|*C)~nI1Clux9qTbgggcOo
zBeR7EySlF|!w()I+FLLkF}`><U9dADJc3L!RuklsHAZo0Fj=(QS*9GEZ2o_7bd4y+
z1xesWZHK??Wua6ec7}1fh>0Aqx@D@I6hJV&WqNCzrEgb>;n1!NEGVc1I!m4(<K2fv
zmUE1`JjigAzH`6RgDdg-c>r#|b;tDO9l96Grt|gf*5LRcX{utquY);HY=*EHRzokp
zLJ^R8Mg|;IM3~Et)f<N^_#71E+y^;&sK#1lJQ_r^7EeNz7-Jp*ge7BlVeSG5kgC+|
z6$=D6TAa+sJwj4Hn$Z$Zif}3ISy?GBuu>6cqzBDyEv5>|>=@xjD^LCaxX@g}PAklN
zE6)S9r+w-7R{^4xxn)l3BXN{yxj~*2E3C?=Q>007>bsTzidB<4zaNO89{HXIv+P={
zfjGlntkBfL>j-k$*`FX-=n!phf#!hr7VI_l6=E3yeGF0zBAn#efik)zEEY*Pf1yBU
zgY6c_aD#EPAUaYT4>&CS4ZfWvvviXCy5h7F#Hg3O0;jTZuRyBKz9jhpS!fMts)xOH
zPmGFiA=$u-<-c9CNVvU!v});OgGqoAvXgPk2cq1`&~)PCh*f>&WU$N?-Gk*cKS7x*
zT=-$*z;H{T<6_du#*p1X!aP_xzd{T{3X$2Zl5|kGFSn4DI7XPui<4D3?7N}Nn0K+@
zYe(EB$Qfhrh(1j9>Rua=YIh9t>fKMjX@boS8cGpr;t8Xh_>B;%+m@WYZoUqbtJaz3
zQH;+85nyP9<A}IYl=ldzhGV;RAzf|^1h7-4iaV?a<|5&njMLW|0#gD#-*fhpo{V5*
z?x>*7Zgfz7Df(k4L9TVz=v+2nv8xd0f%3~Y%Xbq-_vpq0^7-ZOHt?qVT_VgKofU&E
z^W`Pbm!1N6rUbai#kTf;0<wI_^Qvs@#mb%6cSfzPZE>u=C#Fvt{a(>7Jq@O1B^nz5
zQTpWPT)L__>RUx1Zr-IW)OR-of}^NoF}2)5l#9ws&an6%0p<YK?z)tH`xFo}eF#sd
z4~ir{a~Z2)+tea>x>(qZijd4O42aSV$Eq+lij+U{K*6a@8IXH7%qOfS2Mcqh5CQ3(
z!*Qq}7YNtOmEL;XB?L~$91=M-UJy-(V@1RZXbXfZgn28&dtU)rgp=VweK4K8{YkKM
zl9w2*4quc}c>}YPIlh%JwDYu<$?>7S;eyp7aY`E`d2+jhCm5zdxM{f4H5hM-mT*&J
z*%^%arkj*$G)jiuMY=-F15#)nk`XDZQCEWeFh^<Ep$-DGNEep-3P~?Ndqx3aZmoB<
zW3C``87u#E(8i;#0-YOrMkemqU}h1M!lJC<_O2k>ROiP$sUwPNw@~d_v-qYeC|dQ{
z!hc1YnTS<dR2w=2KYSzLJm_lU>@@Y{>cty7Dw4>gscrF+NQdz@i>Wl8*aVQQX3m%r
zb2jChmaFvhetDBKuahz`h&^ah53TWShagM$<u(B@9-(DO17L<|ODWQs*`craxvzdO
z19cHi6U9Y2ZiRy<`P%#XcL-ueNkoI|+X<m?GjO~+tZ#DCqQTKf{8_Ynxr}O);rczo
z;1zmI-4%Lab4XX1RfTuKHJvQIh0}p4yd^ReCyI6;L=N$v=J>5Zmo}|PID@*P3`dW^
zxK_1FpqIH(uDQ0eO-*KK3HM}TCkkm^zOqb}8$xMH|GOY}T&^z+#?_)7v0C$i9OL2|
zq2^2#UsP=7uPc+{=vUn!t{0S>m#rq(R*+3@6bMJCZIyQhUKh!rfr%7;%OceVF4gJ3
zD{T#8p6baj&cqsVvoPn4bc3-#th5b?%Z&255}S(7A#j^TdABP=3{<$zx4&qo+eHXm
zvG>PW!f0Due#QRV@@+O^_9;E#;DF@OA>p}4jOYSgTqVSL(+Yhtc{^y=ZwSv*dy8~N
z43{RY&O?bM=JOM<Z*5OF^GZ4_2{FyMOrX=AI#riCc7SF$$fsC3;5<`+Gn-o*iwi~O
zdSeht-&%4<5bD&5-Sc<#4XZi6<qa?q4vNeqIqE?n?qTD#k+j(!-3f@~*|ADrx8cr3
zh(0z@b>7G95#$o$5Qg``${=!YkGL0$&L)U+;($p~!(&2nzraIB4e_@l-V^A4*4JcY
z%thZYXw-#5Uh9@9yTC3*dOXg6BSh{3ftbgfHQ99RRAEeC+VjkfS424(?i|R7JMP{U
z$cq9u6&SuB3xHLlb4TOK-S~!8oSPx@e-8pKaD*f0mD6`GB6i}UzZTmFFb}vYfF(gm
z<CLZzf8B%MU0rGaZ~@uV6)Wrs=<33OcJBLVty@#5%h^?DM}s`Iwjh_YnjBy4BEYqk
zwuNzAi~Y-ZC|}c^j}rt@Dm<vAp?<4qs#KMd`kK8Ua_i&FpJ6^}?*d^0mr%K}5F+3z
zp*5Q2&z;=jU=)5R#Km9|^EI8eeZX>eq@~71<{qW1&dhCVejmuZU=Jt<e&qC@eE}z?
zn8BJ}KxgQ0wqFs{{*0vToh{J0*5+V}ui^9uVa^5{E{x;4<8Okfp9hK}-josg1IltH
z?082Ah1C;mNLwF>q{H+icXtc?0BVBCXm)jga<eeEGp;*E;+`BvvOPRTgTwQt!pvB%
zW=XNN?fxL;W;k6qMT~n*FxhcC0o#RlM42sCU~0~K_5eVafroPYb@d{4JP<53%Jf-r
zcm={;Lk3592@~KL+OnsoYVQ~3{1M*RnJ|qa=yq0t&GSl^lwE1tbdXS&gH;BVC9-lH
zA~4T|a!G@YEvGe&_*EGg=N#>QGDD5um0-HXh%E@tvIV+a4SWfua_!s>mPd^=qsw#@
zG*{TC#5FbO%o4doOeQDGcLFj-Td4gC`ZlX|`I|PrUWoaDI}@g~*UKQ1IvHRe<ODkD
zBs+Bx9A5HhAr3r6kaBy+P&}t^iiXaS!G->+8`{O^dd~=!7aj3SVJ;r8*3<oUfR#Y^
z#9j>j^b*B9;V=%+!2y%+JmFwmyUvDxrS#0?cKoiPDWFbB3}*^=EqT&`*U#hH5|Phd
zmfi(=in1`5vsc&CQA-OnGrAD3`(t$hW;oZkDzU$43d`xd_Czw<Jhu$eg{^R0C)$}~
ziKZps9U*BJLH)R9pHq<F!g?TS>i|LSZP*%d(4*Uq9dWQ=Cxn|Bgu%E#h<Q3;ZWaG1
z(;KTdi8ve!^%q%apQR1Zl7bkwST`xhnn)+a8=W4hmkFbkh-^J^y9n1ChxI-2SQ#O?
z`IU!--3JI`W?`y;ynBNHQb3HsRqh~!r?sKlK+F_LWk`Lzri+=ei4eEDuBPjC)J+AE
zz=m442Sm7WB`cyu*EsBq8-%;;vNb>}LC(U=!D`Qh^Segen3G_#kv>go4rZK2+$P*v
zle6xX&kI#+&2+!;PKC;YjcXP0Kp6}3u3oYA89{EMs;};?Q3o+ohI-gY$I?Ru!0`=N
zZ+c2#pw`pd6Hf|YRzrD-BEIz?hsR}-U%TB>fa@!x<TRwW73O^9(}u3G?=XnDWzz)2
zNtN*P2Dpb@zb<v9qpP2d08%q9-X|9jXNz{A+F&oeNLt*B1Ulbd-Qhu%A8!kD?W$~_
zI877af@rrwko87E%v)>*F&ASeE6A-gjy!_cpY%3I31;?~f%n7n^zGg<f_tu;L~31;
zg=s854wZW==VmnJK&&pz88fv?#F!$)8RH!;-3}fXL=N?i(O7&A-)>Fr)^frpnyQf6
zV28%UHUgXr`%R`ShTt!9AO<V9Fge@3SODdengoXpw~BNrWG;m9>#IV%L@<^cI(8<V
zuQfrPd0!p>xyy^DV6AP_<Jq}<lP^bi$#Jq4nr|EoM&X8<x_yn{kbpc`@*Zt1g>j2>
zid|k=+*r8lfe6YRMfC3qa|0tkup`k0>zxI<@Y*JDsq7XZ6caZfsq?IO2sD*Jmo;4H
z_BwUJPP9!Zp*R9UQC>6cbR>Q<+is%}8PRdva6BNy9fu+@d2D$ofJ>S5hw7`p5av>H
z+KGHCaK#@2lAEmKw65$b+9{!gM~pj1fHOn%lMwcBjzX*9o_{Fb6m8~l<AY0O&3Ic7
zGmVrt@ot%dj@b~SHad)-X`hyvt+D>$e7mIBt&XSZbR%JIWvqF0qg6YPJ&!0*WCpy8
zBpaB7k`<>>EitYj6YW@)<~fclV=+&$dBujKBhD*Py;{qOrA}kkI<km9h`Ccg+rI^V
z1VoR-HWNpzv9w5Mpjne;Hdf0asx>sM9nmV%T()_FCgY6-rb6=Y$G8afPTu)FP2XmB
z6;Cw7ah3qH8xdIQl52%HOU#<mxOr2M^FXB!PbG4GT!zybI`3!4qDK*qY;lZ$np%WY
zw)>Viszj>EbUBv$F=5;oGCE2-=5>X6aB2G#w~40s?d|RHiN2kt{GZbDUp?BPCrs#f
z6Ak~c@{oq;L6NR6j?i(|`I-P%1)DKW#l)`Tj{$W1%VSt&DA`0L)QsSIPI@2s1O+ri
zYwIPixK)sgL<dJV7q19$7Ob54-sn=t0>T>JMCpiSMK}ZwSn9)7jf}koPGpDLfl<{Z
zC|4J6Y^W2CEMsEc#;GX^l(@Bw$&nI^Vtd?^W9GX!G}l5uaRG3k*dL3FATMNIYLS#k
z9X|!ioWhc&Cx%Oag8~c!-YNlj$)@fT0Td525^ZCY$#Eb|^l0P8aTO}ct3V$+^Sgcq
z2@Cq<r2K<Sbc`bR{_J>uG5?ttSXH&I$3KLcArdGwjP4TTr6)bh3Ey-n{oXQuYCko7
ze=K|gfn6gcaDdGPOvsnaW0?1*jP@5!50kgOrGCkGDkSd(|5Zfh36te;+$6vS&09CQ
zT)0)J1{mf8|EF*FUWCx0IOxP837jTVtA_-T0YkYfe)%g1rzgiKGI^l`&3{sX%D7N^
zc{XPY2zA|c?IuY-B=Kdz=2Anw4XAG(DDO1>Dk%3jnvPYHx9T1c=;~GaRJM0d2B2*C
zsThuDe+?lyrVAsZSpHNfmz2|c^vj8J-9iZ?FPl8mBnm#X%xaC*PUD-T&=_&WO9UN-
zTMaCLa%Z-ikcm~T!PB6O!tb75VA{az%xPXs{#Vl_p)D0I*SH7kb0ok7xq8e+TxFzy
zv27U+AvgKi_(_=s$|G^B62B2fsi#kw8t?1d9HGacd06m_A}`i8ytfU;qJkWz%CR*^
zy(NH=7i|2hfV>#u7A{$kE-B0<)UkwR!Cwe;XPGNIy8o1Phlc5@XYxBmlgIbJ=^Iuf
z9*xY6+eA4g4?E(}LNtZafM#{UDj`R0;nnlc^zHKG(?Q(up5^3NRM(sp7mD&kqsKmJ
z{YB3PF_YPyr)upgh~Yd?o1=U0Tb=`HJ`5lg@IF8ZaLI$=v&^}UKpxfZq1Z_z5p?e$
z+36iAfIh;-@Ui&Hc@Pu<-v#Irq{f{g#2KJu;<%{)0stzVPUn~nR=&_tNS~!p^TQ@W
zGMCwT`Q5>51(Hple2rcw!u=R033Q&c>AUT>psp?Vu<89c!xLntq2wReEE$i?gW;D)
z$IsFH9pPvro?3}Z^zFpCF)d|}d`Vp=)QRib2*PN*SO(N)DW6GfaS;LC3XFO#)x`IO
z(C7>XP8n9YxcHsLQrESAF2osD)8-TlGESz1sz=Y59JgHJ6r?92Ur{a!jhKYOl`eJQ
zHau}G{d>OQRI+>AUj*D9$K*+~%Hg8T5abSSpm1`z@?{0aMBHDO1$MVEnl0^r5O?EB
z0Tic=UVMYTT>uu)E<KO3>E%GV)=1&Vt?feGxtqL685fJ5n73(XPB4-f{uklyAaiT@
zs*Gm@z%HrdhvP?ofFj?%N^d;;N4|3d>w-CQ?k}$ZQf1K+#iJt3NCY+v%#wcQdsh~*
zYWPypgfI`pp#s~ViXm4(7Ngv8+e7hO0q&>Ijl)D!F8UDWSS$S*igBbH*JvHr6zN)U
zI#6ev2*^E;>7!G6woYM`Uzc`ST(!hE1UcJ243wBaA1;w@`6}mn$X^KKg76F{9^yos
z4RXVyW?knhAh^`pIx|kzw}aE?2V%p&7Qtl)ErIGMg5VlkNY2MZc(~CdO=;+wG5R+^
zuTz{F6o*tV2sVdNL!lH;=8wnyy+GLs(j$?wF%A~$B=C!X?=w57`<Gy6hq@ReJBb_{
zUJd9fA-Sjbhg$`iGh;pJ+Tw24fSAV(48w+_Ccr`H-)uAD7$MX>+snr}qG_dMB+!g=
zu7#W^emW`F?&YTIK(XRDs573@CvBV*vhji_Qe!?&^}OkNXx9j<4qg>NRYv?bZYW@w
zlDrqGO%j|ZY;GwcUt!#+pcbYE2YTX1H$u8bgEedn;xYlQQKQ0q*%M3M1ma<%>7uyZ
z`6uCSWqD(Y({6!Q517>!m+CvWC=vli$|wwPxfRe!OIJ-V_~vZ|(!kd=UEz-<!j4!^
zVLm9tNwJ;cx-mBs;&XxI)x!0QHE-uT_j81y3e!hyEX*gBG8@Pa|5?wztsk5kSFv@7
z&!+zXfo^*a`ZX!UlK&_m-1ea(>5HZy@|?*70mqa-v&LXd-yb)Y5Er(RhJLv_iQ&d(
zG14xWp%UK`N;=c0w!{f{@ogSXosR3|3q?~Be1m9Xt46#pL#$46rl*x);kyY%AL&RF
z><*$_dYnC^1BG1$IawTq*5e<d$#Mq%iascV%L9qdi#{tMbU|3ku$8%o7-lLtavlut
z5@z0U^h9@;ka9v9P%}5v1qRu|Tp|g0Qr+KOLYxMtik)(Akp*WzXc3r<t}HQb9y0Zi
z+Bx?Vj>_1~rahh$?P82_JJ2fduRj22R^!%J!jfd&4TVzgPF$7Tpl`-lZ&y67Pxy!O
zhEvCx|AO+EQUf7_rtA363wAcRiICv{-QvX(%y%YW!Ak;N-HysAra*)7h9GzK_PXAS
ziVuY_?sOPSs~++oq+1BvTFz8T8XJe<`07La?%dfb=YzOu0?8=PEfP*I6XbMIk<W=m
zABJXL?XO|LbAkx!saKKu<8o0jqh)4WocoBsr!jseJ?8H=-o$Rz_rwroS>b`FjEJX5
zW3)$)5KkF;LAMe|32+%OdB@mBwEK?rPv~$z73TI4lat?vM$CE~)U{-^%aH)nL;@Y1
z`KH3-MstnjJh@EL9UCd81JO4JlF|Pr!eH?@)^C_yb&7PeG!V};yIdp4r9coH!XFds
zQLOv~s4I;@U!C>s8X=CFRzT#lTgpI<p{APfb|GZPkddAC{7)8fWPhJd(*(hP<=zu(
z{U4Nz%s6F0vB>>Yn1lyb>}Y=q!lg@RfHqk>Tc}!8-i)smp<cw|0~bCONt4dRIm%+s
z@C}Q2y@)YQJ53Rg+Iv=Pqi-k8(@G=FM!IEsqR_lP$`v3U9nn2Sf?ZI$ZkqAqBn5R#
z(U*9|Lp$qvK)n!yWiDnVXDXz5hs2LZ%0?{xED)vU(PZ>1(JInuF&D^Oo83>Fz#OeU
zh!1gGj|+0657p46#Q6f;P-xy|y>!HLAaJ;kn-J}BwW!1@jvc=9JcKKUf{t@0?uO9V
zHNj3)cJQ?r6_C3La;I8$(HEUgJ9h=+ED`SAIF#;=`Cfun6R?rvL4fT=QYI$&POh3E
zKD{j@vrqH2=CF5iBqs5}&h}XK<syg<@sf-+Nti1K&t&1Vvk>!%cQ@0ujz;WL2J1q{
z&sun(5Necn^TTn!05>+Gl%5ArrT!;$q8_}?_s%QOE+#dx5LbT{j4spKIwOvr_-?_t
zNZ*djZn_>nd<`15S7@ewt~wRJ4x!$Rk3?q{qk`w|&FcVe3i+)=sglkc?iQT;fu1)?
z=5Y6wpk7fONjs5s-Y8NWulHjsaj5`Wy1R$dw?Acq1I2pWl8IR}Sf-yBk#nHjx*E#6
z;co(FMx)Eai&;V(46ltkzoQ@TtYGFMSM`{PlJ;cp5@njkpK=m@zxHp!<{1#H$k<S{
z<I1Bd2Mt&y#B`y~7_AK^M)8^uXN*>safKQ&THh*yrPAO7MTnaUQMijUP2IEtqY1dd
zijB>3Z=2!-lyic~qMWSW8NtahJC4l+I(HfEvL<iv4#-41aXcG$3dr%aaBE1%PyPd#
zcTMOzSdC=u+8%!>ab{7}w!F(BlgEV9G`v1ki+dEvgG=^U5?=l(NYm1+miW9(1C!-J
zJWDVwQ;^1cML~F02`A+gC7J?VTy7L{v+523j6|LM(Crp^ACipGsJF#oBFr)bDH+sr
zY*S)Q86q(_V(tfo%Wc9A4)#AQgtla5(^*?A@FBFTz#5o@!_@@2BvPzl*D8Rlxw}>8
zY7>MTpZ1fTwvrhU>WpwQq>1H6LNqI~-Jo<wi*$*RV3-s+vN-3zV5G}sN8VNv;Q*`*
z+*D!)UiyDvZZVw_;Gv%r$z=q(0H`LCf)80Jp)NqX4EAxOq;Reoo9N_Cbw5FriFs-g
z4<QbUbeusheKA|UDa1W}yfVhA&&Nd?4ZT!CMJj?Eq<0i|7te`yA}H0gRsF^%1p<>L
zuWum49A%@z-hKa10bDWmXz7w8FI|0B#6uR5PT(yeo>_FDfIHPR9-ChPn^m0uyI4mG
z%bgq0ijSWN%&o#v3Kd-HOS>eN>rKV@g9!TS4E!%&sqYDHRk~$u_=O<H=I)A=5G&3H
z;(GYuvK<BFnW55+Hz(1#m07aVTWf>!7Z}dFJM|PwoGr|`Ve5c08~slSC69QBqjw8(
z0fwYQ?1&W?DC5z8ISG$TONiruWVE~Yf`powpJbAbexM^pz6_c>7}AMOZsQVROfwv^
zvjzI6D92+C(HXmpl%Fr-F|5*Uig-m>9yPKxhg+|f!FZZ?6p;=02MZO!*y6HG55)!x
z1Gz~jZhlDCXcMd@hr=BnXB1_&Vn5UwM`apq076|Y9v5xKj5L$qui-^N=-Jr6;0P$@
zUldZa6fVb1xLWBr0DiM8x+4eS7@-pHh%j%kDw3$m(ehWo+@ri%(K!EsK+0JkZ*c4M
zZc*+J9I-IxwtN+U9#uz9p0XI<&X$F#o?p+{NvLzuEtmA>bHcox)n!ns(D1{^M<}r8
zyw+HLaR@U7!?M0OOtdp+a3C(;EWk;RaB!PEHNGIs4TZDo^<^&fm9GI6dNO9$WFk?E
z>B1eI>k_&P!d7&NBCLFtaueMbfnOJ%c{A*1@P^~kB>|mRlKYqhZxS>yK493BqKT*d
zGTRL=x}Ri_S!tA852NyFAJdj1B%Ew(X^C_74bz!TP;sojG}J^}4KmqbI&iJ9iMGN{
zBOOg7(7mLARt~HGgt&xs*$(;Dct|mGPaf@0CwluYQ)EC#&|!La^a@iE>O3nF?}_p}
z+FPr}0?Ycxsp%fZCZe2e^4G^`7vkX6@y@t7Ae|-4YTDbpxm*!v>eSRq?+M9Wiz%`%
zzO}p);o2>Vk~mVNYs{+=G6$hgufIY8rkhboy4p~nM;v`iI!iioBfc$oqL`@jar7mG
zI&y@9%yC8jUG6NkIqrx*2{Ge)B*JSv+$YS5vwYT(Pk0ieBc`qhoM#fYr_8*vZQ4et
zD=yc3GRiqwkkgbblYBFtEsS2TTP$&783!xvbbR=%Aa^RRl2_ySm7vWkR4Dk`zd?X`
z!;X*VSTxnmS{abqG5X==?V{*jv$$RSy1sKiv0u?0Rsl(~2@~U394#{UIGyKe8*rj9
z_0OrTvG%Hbdql}VHC?usNOOzuG<>Jcj6GI^aH%W0oOg_9v#3WZCONAdiF1WIE5s#)
zD$07Eu-t3uf6}jBA;>u<pYao(RqiUIv%-0-J}7LWrzDw{ruaahbH`vuzCSeOUG)u6
z%7~*o3_Vc{uK^9uP+}@mQB<c-7fIh*69T3oYv@sCl3ZkxBXghv7e=C6G)$Bx8g1=y
zlXC=VU}V)1u@;ESFjDD7xYWdK)EtlwKP~v1E2OK?&Qd1B3w+<q6P<vtMe2o%g!&GB
zqlx5|6i;b0zWCb3FWxbtytet)0s6(svs$L*Y$X2pO+Xr$s}Vf4#J5|3;o88pZk#7b
z?a#wCm#@b+HAU^NQMkbRkgfqTl?__f6b2`m<HmbqwkT)FdWQy5(%=aKNwE)G|MkDc
zH>^gBQHvcngrHS)BB-Y}ju!3y%B@<QPsK&^AgIikuf9CRambHN$3Pq*I%`akjj{IA
z4gk4v(N=zNVHqk2-b9`UgL5d{8R*|tI2T!mXuve_wgM>`TH|CrwvT9cOWgWvlD7lC
zM}+4doR69pZz~|_0o8OH333_Pav;;QKS?dOm41L5J$)=8JBrB5UUQhUy4?i026!&<
zv~-g&&$yFVK>kt2VTMPi^?(2m#;G$<%)Tkw1!V7*o{jlXkhv^pBt4z+<7q`89Cjr;
zrPl;G9r%triJ8zY97hry($AU&KtV8Blt{U=$lP=o$S`Vl5#m9h-KM%L;?OZdJ+1ce
z#2y2=gk4qkB67Xg;h=69+i7qG%Aiwafp0X>KjFyiSf%RHaRC;$7{4qt$7z-h8+3!|
zDxqeE-0SetP~1}nOnMgVCs>1?5bE0C%4m$8-E(DBt{`xPHXLsYa{2VGFcT`KE&Fd&
z#Dvx{?!iJ_Vx&yj!pbCW;f+CEV)=!t#xw!BY&yOp%nO1v5_!mJ!A<#giL`ZNR3#+M
z!H7Ov>t}ZhMpm36lHYiyyC*jOHiT1_t2O*p#iE-5F=%<WMng!H%fbaBdjIn!fbE<H
z-FFD#+;wb#UFQlyT={`^wCY@!?~l_2x)t$3&0dh|@AO?Tcci`w_XPJ5t<`F%PItIe
zl*?V^rp%)vbB{;*O3J=zn}eBu+Q!atKbj%bWk&0P((Ne$<fHAR*mKPm(BvaE)!cYk
zl)Ig_W!Z;dJS#XCj?H%Lyd^ZL%F2t=*^$_BD*$J%Ys(B;O&JxT<|r$$uARlN1bKpB
zlZKKrixxlK8r-vXw##Pdr!N1resNuC1>actTo^6h+A=*(*@kb|5GO-udw#Gj0FBy>
z@;WXM;gWMnzB^tP?G|ETk<NJ0?LeG9lV69PM~k__TyRzpMg>pR2yzSb)y8AP?ID$k
zY}C)p1Uot2O%_28T0fiU!JF~l9YDllbj|hHbVn$cSK&M2-vV3!j(WT5@<_YnPGD{a
ztSq$L$&b8Y_a<gw<`1cPQdp8tPrSxbis>NQ_%WQ>E+x>lMyA4jijh1&u(i!@371u5
z*ISoGxtJHL3(BQq8BfKSD~ueG8<^6L6lKm#LYUfqXGl0B|4H%X*+pzA%F-5ghOpcO
zddaIjb}U25hy=@r!Prfh*@TB(c4!xu(RgNFM^jG-AtyvYY+^)E3ZzwCFSu~mB9nd$
zAQ5NR-OT}55V^t{+lX}f<H_aeuLQVQGA$;JKb8n`r%XMc7GnNr8q*oT;P-&cNW6n1
zfe^VFrwa8-#>7z9`GxKl<{z9wzqY?*b48hgkAaiK^MFv*r<q>hSz~uddVEXU<mlG7
z(_r~yq&4E{{Bu`xG-mJNn0n`#W1$N~x`5IiCVf#Y9@rDq<AR&0^>|J+yzFBG7RT)c
z0dF`GVl*&GEWEb?cvt0djL|q<fTOcy)#E|YYV-k`qIDk=EEG7r5#j#B?OkbE+v9X$
zt^fibEr!EBLAjKywR(z?e!J%PK|Nbkd$0#eibC3<A>3`HZr0w22ZT5q46vE_8HrB|
zqnmak()7f;WgLdR*5dws`JF<e#2SpH_JeSZSjfg=RJ6+|lNE{PhY50*(VIm>G2i}<
z!#I|p5~ql!xN;rK{Ku-zn#E|oOSmH<YXjUTfReGBMqReg0Z=Y`vQ|)D7YcK6w0VkE
z4}{i?H*;!iuWwHdoZPYlf&~b2R|tkF>FO>6b0n_J2+h{dgB;BMe=8AQk{iG|;=@AB
zcof;(Zc1AE11nC8n6#N@sf&cTkTUb(CRj5b5~fZ$b#m-b<=X{jHLpZXwCAC8NQhih
z<Hi$U?zy#Uk6seEA;;tW&LqBnD7t9K_I}v~Ufls^{&nHS0cW)(fMSKixm{|6Aa`AK
z8^#Q=FVmD?qC`9`8iuo<;!Ni~QF$7bR#8Uh@u9$pX{<Gl?u2v~!@n?>BVzZi0?`d!
zim{InmsLK<2Bf*_6$VdqJn?gVn<vb~wRkxHHiU`tWBN8L=3;PG??SK9T?9`r-Axfq
z|Ku4qq+Su`0S!!tJvE1xh6^Ud<7I4|Bn(Dh4-s5}bWdkE*6amwaoPCTHSly{Fs8j5
zyV7{D48i1tVwFm+(MJeZqOXpv!GLH=HC2a_U+Z^ttcek-w-@D(iR@r2PBwlI4*<K1
zv2&2;9SH{s&07U#g@&&(=Fl!TcHQlmNu`i(F4V9ZSqz$|l&CAi<XXdyt|QhP1anl>
z3+kbpl*m96b!kV`gg74Bdo;>~Y6x>%=uIjOou8Eu`)i=V*t||CR|9n*ClblrtuB=I
zo{q1}mV=74k;>R+P^Q_!VAqID5nd88(Z^=Smxdj;tz~k2Pv886@=^W)cWXd;0g*tJ
zJUJ(W%|GdfyuQ<MSyt!^eIo^ku#Tot!a@Sw8u$g4sFoI%1iP}M_-&VqiUj2b(tOWu
zj&-SDiJF_Nz~a-O5T}TT0rE+g4dXb$^bq99<W2IQWkgf~Rm!*FSdmmOS13vR1R=yp
zuDj@EH;ATMvdgZs-$~oP=LNc`LwZ`Cc3FH*krItUuZ<m=AoSq=<ivPaQLajDj=XuM
zIv*g^39+?H$1&}K%ttOlk8rV?&hc4Ji_y(fxEstRh<ko*E|<l6Mbp%6GbhJY`gV#G
ziz73xWE>1cOJi_95MLLOJ4^Bgjp#ERs|t1J!?~HfUw6eUVNMA>$(W$M1)0x?ygCX>
zBIw70-5>~xdj4jgLqN2SuoJ|klPIU2?B=ACxk^}0ol{TkvR@M9nFNuXzV?m)v!Y+G
zhhxCkh}xk=C{{6+mzM=lBW54@)6UZQ+J}KUeNa~D!Pr3%Y|9=l;*X*!Bq!mRUL1co
zlyhg(&!Rbe1b|D;ks|3bW1KC_sY)LsC%)&F!4M;CclmmWr5C_t>N@#IqI!tQc}TlR
zr!J#f?v&!2r^M1j(_;BrKPqz6qa<<$IsGUgH$P1}5~qtc`(dQCWvd=tL`wHUNoPQi
zXEk>GGC#31gx7@AcV@Lti9L@g5|WQGTIpgT?kx=e9*p)ifHyuC)a@dtVYn~ilGP>x
z9aMa&4Rfr_wf-_FuEXgL;%oxVMXp}bZfrgj&ONK}+loMzV2709g{J*S6w-X`<J9K1
zpFq;N2Xz6r?KlW?f=vfzb}{K^Am&x47SnFgYV+|57T+I<BCQ$Il4kqT<00Jz*ecSs
z#!di$IV~6>|3u#tlCu;N)``^OH2vVBPnh1S;hqv)Fo|~|q)T{4V6Kn$qSWVw65>83
z4!GLmRY5MtU=J_e#>zhj$Spi7*L`tziJ(~0t;P?8IBA^?D7U?T0p!Zd%~t$IbZ#OH
zmGUi$M}<zb2AUzHvCjp$bLt{byDT4ve+lOL;22YiIuxaM3iO&gE0#PF!o`+*o1VBv
zfQ!xD2OYrPBt$dS^l9;tzTJ@hjnUZsWJni|rGg_-Of`n%aG}n%E4>Ya&(}Dj42o#T
z8k96%33Z`)PDOn+hjiU^RZ7>oPF7?O1SuPF@@<Q#-wMwIl4&;Wq3#w&5oN&9-HbOy
zx;}_L*fSY%`q(hjc!RO>DMdQ+KisHt%KV`)bA~tR8A0p*+6f~D_V$cMkI39_`zz!0
zio=Asx3O{K{36|W;fHf_Oaw}8>ibTeCuKGRaioBWGRkHmZW2P5ml_S3<uoW)GbxXB
z8gSkjU}k-z#?;Fp6&-l(GmDhQm@$a^O(Cu=8=howxo#PV;m*5BF|!P$rxkP)Yo9U@
zSA2)jJUk@C48iW2j<ow(Mbap!(%Sf(Acq;iZDTXW&MtrvJYujFKL@}gPbwkiP%cDa
z2qxGOIY4A@d%ggA-3&ybzvS;$e2)J$e@~mz7T-9JFy@=CQqy;i5~8WXFU|f2%5~?J
zmn5A%A;_#m4}vc}+T%%K?yp*pcpn3Oz#9U!@W>%H<9o65%Rs0Y^^Vm@Y$}lKMsT87
ziLFF2usABMM^l9J)_rw+-Nef!gn$pDd*Yr890lv_@%TuDhkjRmY$Vpc5SpHx+>eYz
zRitany#rQ_`2BA|^3vYKr!G5exeR?raOR~vOh@PM09+Q`qUdKmdiElq|HsvNfLT@)
zZy!N23IYWZCF~-ZjkCK%m7E1!B;%c#J2Q9E9p+BhJ0uAr8AL=71VIszAW8;NlpvWu
zIjZCwM9D$E-`j7`IpfE(56|>_yZdxkS9e!eS5@aZs^DIjp7irvPyx!uL#DR`-4<v@
z8y*054B77sb(OqSs@tYGQc#}Dib*Jm;q0<T*&ynC-Wz-VmW*y+HxgPi1_Zb=Y!&f6
zXpfN+N^wtE8P~tGC{`f`HvA_5f?#o_U6%2<Ux=2fsWakj{cURY$Rg&6%ZfDd3M#y>
zL)uH(Bs70>m2@}R36p}H_TT<HXTk|v@2kJ2q`OGmEMJJNcLkUS0m&*k*J~!OFW5^K
zf{b)C$ie8USN;RI`^q#+0atet<QiDGWq^{RxBl|VA}1@Rd_m(6f@rhjWW_5=`Wq?e
z0}A0(a+#~Z%_*bUbwJ_Y2{Fl&5p=wMH2{>FI%jfBy@tQtq&|$J(BQ>WB}B`)%^Qqm
zuO*iWG{7AILX8}6HWBJIPeHHHwzMk1BaeORhs>K*uG!e!B<0*H*bRXlco+WJ>j0QS
zy)~JD{Z}auT8-Da;&Y`<a7mL%n0)?vF!#>a46)!1AZBJi8A3Z^x<I!#5hq2&hz_BA
z4)#XHzUz&o_*}PQuF$%pm4+(QXUvLk{E@#++`7C1(wPUA*!d=Ke+;`k77JPHohopg
zF*q7(#W+ov$vTRuj+o+#G8>YXZf7cSwJ>)F4nc;AMg9b0B5)=fiS9p>HqD6ehP~6J
z0$ejvK2hWFNV&Nv!!-$SyDIHg!c%f(<@4tWq<j6X*s}eFzhNcWBGK5@|4Ndzi^r0B
z<5VRwOLG^2;M7{9&L8DGsC?(e-;@xkFc5#{7XEg7bmh*CODu9LklT!|5_{zk;0Ez}
ztnM*Upl>74ih5}_v0LT5n8eKqzp#;Nb;33@EzH}N6{XOpY2RrR=oYYBCF8Rn38F{!
z5v*G-xs8O0tT+WU@V`P_0{vhzoA~DK1tb-#7KuIo2IN+AY{kk$QdQhlRK{dc@SX1q
zp)!O+-atn8#MASFAON=YLkXDgZh*Xw_G8n;W6f+>3H1k10_lk*mYIc=723`qCihbM
z+wH_NW<&v`Ly*hp@>J@<YOEp<#_p=?hHcBT6bj39G~8cMUKEf{c+&WKA$*48aC}l0
zz=w{0@?!BlMFH8qa}M2Ch}+^<0NUb2L2e6yv$Bo+g97s;hx>T?L4a2YPoqw$I9ZpT
zzLy`wb~SWj8{JnFYbWe+h4*Fv^7pV8Rmax5A1F_L&WFg*o0kZY^>s)1st|uZy;c^x
zJx~<V=a~tI34vNEetul4lxHuV5csvB^t(xzhf7hpvV4_LcXt#ID1~RpSzVkbu)ETu
zs((^(H(fCRkg4L!4+6Pt2cfTc-<ExDX@Sl+s>=p;>oo*<_Gx$r70d@IWsjUQpWd|U
z93lwnReHD`IjyWoiV_U><3d4RNc1{Z+^BS3ll2Y*kELjHcto&i$^{_<7OOr~Agp|H
z8oyEqpPD)w$Bz$`Fo7LD_fr94*eTOvgNKVBvFIe+(_8_xLff>Izr#6=94)yc9#)#M
z;C_J(<!z<-ke*_TQy(GWmJMSggxGtL5P!OXx|^FE0eM82v<maqi2pna=)q$qk3UYg
z?PH)Uc~a5R$?KJLRoJ6&M7djlzovbVL+{kbL1?;+n6bBx-Ib<s91=#OO9_9uI!}+E
zYl%HXpm{+row6Pb8uLz~_kjCRVW*%IDX+(Vs8u2EYR;OWIxesXd54uX@+Ki~BhkNO
zg*jUv5%c9~aMA>bC0lQ-og&EJ?W^kpV_ree7M{kEM|K}B%PF>l`mo58MeVFC2=;gm
z2{I>fpHYvSl+GN$YR-c@0-$Z8R3+AViX>yIpl<c}sZypCK~&|=!`A-3Kvy}y%Zahc
z)1*x~CMlyxtxpQ`h$v1iTH!7J4no@rh{-zn4-)3*jtZh_RwL~rI8z59LS5=FAvr?m
z`$K}LiKsP$af1>xzhgL!%(nj|>9+9n6!I9e{5#Kp(WMT=Wg^BX<!_*6OjG{aoPjMG
zGpoCm5<wWA4xXS=(gdTa(BusP>GSZ_e$N)Q$=Mc#SRC@4%l8xA88yJMO8R?D+^XUx
zJ|xHl?86^2J}a{*PKnGB8_$zPn>0r5ztrEfp9#Ww>j|Y?xV6&7dkad_TqfmkZ(j_&
zNRkDcm5(=mYv~A-7VH!B1GV(<?)X^zRVhlbLMmwJy-J$l*gS0*qyxzpUjp}Z96*mv
z({zGeP^h_rHH}V5pZFDlu0ks{&nDa1<Wk|qWW_K4vkb`fHF>)(`YJgfC}usiSS%;n
zhY>|zR3fi$(%ZMigRg<mZXUYo=l=RhB_Rfa4tHPe@pq*&zozG3#6n+vqXZ?XaJn9h
zE`eqg9wy<knbP+NGgo3rhT%X>j|dtbUoMC`mV6FVUN;Ta!+taFQ|ZjSM5xE2GoBaZ
zu^DCiZ+VksUZ!w0<%@?1VVI}QnH5LtZwQr6vi+>?7YlF`p}jT}%vjzkF`|j48;Q4t
zK?HVo*<8djZ#xY~^lNoYX|3_-)mAEh>*Wo@Sm9qq-nQfgue6(pdxBn#IZX)7@@*#O
zF4qb$l@t|&DbLbTji3Gp*i_<O$X*7QshU9NO-z9u5sYcrUm4{Nibe1ol^&O|Fgx~f
zXr*^SoeyU2f?&G}F_CrP(J}X~5@I&<AYM;ADadqh^bE#_O2S|q?<+WZkT5?Y8Zo>-
z65uK^V@b-b8w8CrI8xe>;@8ah0MHW;qgS5p`LIZ1jM*9QEA5IqP>$52>mz{NRur+i
z+uPw|AdAiIIyNzNSfs^!`|6j>J#Ac#LHVj6w?YpB4yA)g0?ibdQede}*$!nimC<|l
za@vyPNYn(6XC+jOczHya>8#k6ImZ6a%AB-(BrQ_Qeo}lL^)yGoc35(nK+~Y7A6fi^
zPm4@CswJDuIN-m4?uy<WV%dl*KL>&~tYFf0RN{RBv<;VX#$O)(ow4eCyNy@QPQw+W
z(u^V}C7sIF`U1(^bT(po2tlGns5?0#e~*5i*_ePe(A6LkS7R&Xg9oN>jOzW$vQUF-
zGLE>talf$KYK+>n(Ofw{(D;01Lv6(4LbNQ;m=UKg;2PO_2jem&JXTto{5nn77XeKz
z4EKiPG^L$s{57p=d}%>IddcG%l5fY9<O_W~)qR=%raXMER}<q3r9I9H??s@yB)L4A
zvq>z&v?Fd@h-`GVK||tRCEN$OnIW1zoe?SVIN$Tg&EpG`$6ePcQe*L%AQL;C!EGhB
z_adO~5CQjl;s~Wpd|7a$_g~%@<_}^C-p4$MMM1nXh~cVz3OYqYD#F>3lBI2&t&|(V
zomMM*R-7xyGm4!Lf56MiJiY8w;)=<Om3dfmi9Ia9ZA9nZ8Lueqo}f~Z7EE?U?+Vqz
zJ9}FEU~&F-FS5r?d)z6^t+DYwR_jXuxivh>z+wI)0b&N+?brU&_}?w@sQxwu&^K~~
z!=(bm0|y(;lU<5THJ%Y{e(0-77?g2Q0{vxP5Fz+_hMbQ%&%UM;c*^JIw074qY=1+w
zxga;=aAjL8yJP`L6)HQojZ287H^n5|OOR_K{s=|}R|<fH{Z;NQDD#}s?$D4PR<Fc<
zOF0MQDa}<3DD4Gz(zIDIV`-A6IqJ@X<o?9sF;l3&gC))&(XIrzUSwl!4FAYkWJH_R
zgck(3Q;n*(<NcN`sv$}QmwQJGa5ar#c^Ag@LQF8-P3r}LzPLULkS&%IjWm%7bW7Qw
zy6KC?U_#ckZ6VF$l&$i)DMS>gq3nN9Q2uhYA?fmQA$e4i26xVKMHWQ+G2&Qv31NC-
zDN76*CE%esX!CYlp1=J87SeRc>@6V_naRWHbwQMvHFH)R{#E`q=W3Udt}B*Wp{NCe
zP^=HQ&{`3QiH4Lyn3mgCBIW5geuKu*hWB4r1~-#-5N=^y9hhK$gUiR{G!oaBSz+4-
z@zNA^BpwuOVlu#4c^a__s3+toHd*mzna0|uCl*|l-{GD5yrj#Stz`Zl?c>0Q+Dw?~
zthadBj2Hb{Q7O8&j@V%}(xx3M_M|qc3UX`YFjQ+K``EKq2Nl7mOpYbi;BQyU(Lm9?
zHWcFFgIKk`m|Ftv_^<TGQvw*h+SphvHeQp2`HZuPylfbNy#!`9=b`j!>@UDQN)It>
zeH|87tceXYZ4!kEha=p?f*psWSynbwna9F)mjHLPufqI=3pbur)ZfP$L=P0kkA%5v
zIDklFdSQuRyeG5moz@}`gs<@A6^G76)+U)*7rzg#<OR?c^ln&-4#bpooMxXHtl(_5
zjneLYKiADL8{6o<0^L5YmU?Jm^a!H=Wju?aIJPV+KM{gx(zoxfTU3bE5Z)x3p$Rng
zSg+cW(a=81Xp9Grt)4%k#5BEi4vlk4Offm7%a6x|xtjw-I>k<LJpiv=1ASbAD`^o;
zM_w*BI|;5RdF%xJlE+53fiEa+8snX)2{31U5O<zKRDGb)jAb?e^5@Xx5Cm&|0Zc{&
zEo_FbRtk1vAMTH9%Dmj=!401ZplRKZjsY9qkfcAzTkkR}dS?oVwxH>A;ww{K6_3$3
zVg)5kA-4TilpNLAti)Kn4jPulT*5qtsBkgzlWH@8UI|e3Gjo#jOth;!ADh`SIePWC
zX~DGWiFx_AX+#AY)8B+(8R?EMP2*1!ggcXDy0eKOccO`BDVl*D%S`OhDF{zpkekmM
zZ^Oo4mDn(W@Cn0&`;suI+u)KS`JpX6y(lO{pmx~}P7>xmaxE>N6=;2gK#zK7|8RV)
zq+7*eh&C@K%>Z%J(`!R8z?w6G+$t<2y$$au%-Q5Tr-Q(yf*4b7t?RLD3kg?@YLp1G
zv6c{X1d5NgS`3tc0;SjEQUR`$MKm1`mYL-`<v$;tO3?P!nl3S7o|1Vi2q_UyDxEnT
zJtPq#X9Ktmqikg`>LNnCHa42Pw$4ogi^e3Od0JHvp7eI(WMSq7oOswLY^0cASBcub
z4b$Q2%IH>1n}Q3*+_Eywy`*UA5N4te^6FD_4oMG815J{n>ps7+^Kh-hop-#bq&R%W
zoM>!P<d>bO!%O^FAayFZ1wA`mN&1i%t+caUAdET&2rGElru^-O^X~f8DbwOFo4Ewb
z8ah8VJ}A8E=3ul;hKdU7kmS`}1o9=l#fqN!5G6B@vi)%{GB1|e0xXlIQbi_?WrcX5
z*HJ^%lLkFjFLNeE;Al({B5_!Uy4XdjalRTBF`}!4bkQh5<;w9gdL23*P)dtQ7gs^m
zZ;){FvLg*I_X*1zG{-Ww=n9<P{>>t1Pfammk=33M=8iCDw6^W@Es)Grs6m_2+|pU(
zc}<TfwBc4oA!K?vgd8OV${&O;(k1%a6%dh%;OH+1$ZW33%Y0sKYao~?nSj9B-%{EQ
zCQ2Cb!;>fLe}$TFFjSXf0zxOyEk-)4)348!%@n8(;IE<>V9Rd<CQ)b@+qqb5+ad=>
zrfeik@<&RH$%kHv-TE0}uDf57>R|j=h#ST#kI{(cP%OROgnXRX2x=+BePl}-QJBun
zw+Et+=$+H?ZM7YWEIet;$#EdA6+|nMqS&C`SBe4+If+A*b#^4}DXG5Wj*;s5n_zQV
zS93t#94roS+S@yUn_7%x@-)ZTPoUc+m1Co|8T0Q91oyPpurdfG@?iB#M>$IX%s<3s
z2ZMjHlBO+(p}wSBSZx<D*9m{72)rlm3Y6EGVJtq8QW1zw6-xKGcjkEbu2K*evdUl;
zJ{tZu2rX&V{lzM~lkk@a9Phn8VQ!uLj~g*VnY2nwpT<ix{GET)<zp<lM^S^8&o+f|
zstTh!x{t>IWJpO5aJOB=Ul3xdaj3=|SbC)dx-C4C%6W~eu7uIp9wuZg@EuB-1O$;y
zmJRC)$`vK=C#`BT1?FotRD0}7c?x|)Ke+KcFv}UZUe+k<EBvQp&fMZ7=s`Fn_Q#j^
zEQ%@4AyQKbaL>rG_#Xn?GSog)L|pp4fEWgLI~I@al=AxBr_dDFDNVn4_k%~U_bEzu
zVfNFD8uuwd9$MVJ)7}oVCDar`x=L@G#Y<&jW*H`K?XkqZ6gE+~3dRUlCIm|gbd8)e
zw3=(~tygA+{S`|X5*#4V^aH^q1ib<(o8CA?KZtek9EqEhfGcM3SnbRD+YM8Q%rsp$
z*$>PgMARFwH*lOJH1m&wy2U91%p6>kqMwh`%Pa^2OwpSr0&Li^!2U%gh;~F0Us!;9
z%#o>Ip8D1JxXh&|zU%R;1Bzl8MR9byQ-FsJd9g=*Klwl)dNpm%?6^gL`)3(^5*Rsl
zr~oml@E==xMJcZp)p=<lPvXQQg57h}5qc;<jZUaLIx4g7Q6r{WQtnAQd{kwP5n4;n
ztr6;0nA<U0^*QeEf;1&(PKza5`P&t9!_ImWQ-rt@?27yNT2E|R<^q$lVh16vRzbzY
z4qbxip5oxP#Suz*te6S}5q_)$`iEI>4Y;Q^`AGg(OSVzYC`%P{h?54pk5KAvpp>k}
zmP)Zi;Cv!&>b^?454>K3qBzbJ;?}X&C%N^CGE-YU88f_4B6t+HGo-!k6f5XQBon@T
zoG@3;ny<a|mMRE6W1B|MWPiI;fEvfM9nILPqbz{X!r&bxB(X(Hyxv*-DwBT(?ZU1C
z;TlXfk53A5GjQ{-4axrVRbeJ4l4yM_X4U}QMX3RG<oI(55w0x7x&MnGw;r(zK8;U>
zFb(SjFpl}UNzn2hR_dWRA*a~Puoyl`DQ{Y+iF7lQB)hn;7o63|R+DY9VBKl7&gmWM
zgOv7k^1CRa*?9uZ>{2x9PE^mIJtWwRMcOUwM8K6*`eA%&=F?~vniD|@b#_CXkh8ss
z2#`WYKB9U}LB51`r&5o{g~2o_7D`t<sif&w>0~4z&9VbPZY1twX#*#cN4zfBG=%w?
zkFnT6ARdf%9w@5E?gHFg!bi)G60OY&gGF^*(<^2{l2vaL?CLnY4H3+3@u8wL(GWQ4
zN!WR81kmln(g~|do|+Tn_9Y_*E-5tco)GMhGvQ^?16REw&=f!bM5v?uj-z01HnRvS
zCjiErLv*F%^N*uQ0!hw1Q-B+Q_8!(D|2!brWe9E2Q2Z;Wc>b43@atw#d4o&ufp|)Q
zRv9kDmpYigVY;caXHJh!CHP^+?3UQz5dKc{T6@`UN_p<{7$WM_hGdSWPzcsqZ~Do`
zSnf~|%|K2XO`a@{djv4HDRwokM@pI^a9i7Os~Vrc6&71shl%8oRR+>cKhX+Zdp6=7
zmG%Jfl4~8U>&HS&if&G`G6Pxaup$#{66->IO^CY1vpKW$w+mnmf(uQ1eBp2)nwTED
zug7*uI){S65hgs^N=Q!qiX64__sQj*e~@K1&R5!{q=b~k#Os0}En#^&<F+G6xZ{=n
zd0A$AU0^=va2|xHNP5t9?vdc`XJ^{e<6<GEKLVg07}eu~FCSf?JR>ZLM0-s_^Az(t
zT|6wIoW{6}Ma+fi-Y!dXSu#`|K_02cia#j6oz{cFm@dR!?HNWXHoMG$p`Xkl4iGZl
zIR+lVjL-sQwxK4w_fX6@hHMa;ah^1FPV_37g(r@9ai7xewPKVbwD0yq5Kjwi*I5Vl
zEdd1~YmL7Ma2bRFj6ULT$AZ8y#1X~u>`A5kdxN(~v|I}j^W%y1mE-u^R1r-vW<xt7
z%zZ&cJ2;fi7MmPjWMm1@jKhXNm`5C`BWX|fJE6!X)1u@laiK8Rr?AxYjD_J{fnG1L
zI7lXkIBFYqe|wvX3%1LNMTL4DhXAbYafUF+#=F+(qTzO>>Cns>lVkIf_}dl8nYt3Y
z3-H%rW~q}9hRzj8i7rfVetv3Ef+>tZ@reM@hZiP(^dtUu7l~}jnnt9(5|Wp`RFzMi
zMlSOv3I;CI($>1bk3qeGh>UV}Kwv=H{9^e7tFgjZWka1M^aEoB+jVo&WP4s(h+9uU
zLez>n9xp2}ub*6HB$gi###F<2gjdH-C*f9iqKuR~`uaa9Al7=BSjSXBZnc!D_@8U^
z_ZKSe!91fYPCSE@Kh<6*c8%g*taK)rIf{E@&GQq_axM<g938QygL9Yv85k{>y`o|m
za)-z{T43I4rFmgIbr?#p+pAMWFGge&a$;1WCw`Mo=BK$<CQw#7U=jFpf#WP*!+j~P
z7vip^tx_s>ym-zAb!W1xZzGmD2h0SvJ+t&9lLcn_!Nc-sTkz*V;;|_+V~+mzz{*oI
zS)8TT{Y*db0W>EK`j(6Wo)P9L&wI`BoRV%C=d!fx!Vx3qfkJd5Y{5<+|H9SKNzA$k
z`~f4Ze?HhaiQ1BH{)U3w^rYlZcXv|-X5PTU3<uA_*sLszMk5J!TMEi!C@)JcOG+fI
zBhC1Gi4Y}CK`EaV0+Y2O`o`wJB0-xa#PZCoQdGz}Ejgz&<18Vx*cKd#x6Ts;EjXts
z8tV!dItTqkR7uxVp9mVCzKB+_*+l?uD3V4iE8rGFGTS2Yz|GS4Z}S^a6Qc)ha05kQ
z2EDt$29is3U5Z(~GMh-;e{tGJ+`C&C13Y8KOkTX;@7xNWn?Oq>KtIl&8CU(5zrAQ9
z6Zga;N;`9d5M$N&)g=IQT(2&1Ak}vEn@bJRQzi(j#!&2W8IZfdvc?H59YkLc>dLW+
z!vUd8^O{;SCSA_&X-6eg!s5Rxs%S;arN=_y@;0HFEtqueUAzNnaD3vSo?^WwcQlnZ
zbIWk2ue#1JYwbgc$*Lgv>@Ot>Q*{#FMWLQ%T|s$sWzu@_`P&4!N)+ZO#}Gf}{Jy}X
zb7J@Cj5~#8vT3D`rv-Rc@fZU~9yCTtKlp!t;0IHRdp(++w0YfEf|^o1^V=B>rKwM@
zAqw#w^IrwPm`!h)8VBod+R4HymuG$dRzaSZqiv%+VnGtZ67OR$u2I5$K>dcT4U)!@
z*MPYdXjHJqO-uO%w1MFs+$HHVq2@cjXmbe>@if(8X71+hie>I%A^wt1aBwL>4TPC8
zU7Q9R@nTs_rvdKJJ{DrSH;CV>{;Yc~kcVB5^YW4};0%EfiBq`D1Y*~mh7aYC7pExg
z&kt5x5iuFSk?YF3YxB~PWryp5Amyy7ljCRl+eMmOw{v^Z6MNnOmbq4Uv&Sn<{+ZLK
z#~u3H7h~gcTMke8Z~a0)u>)%&-1fzpHv*VE?KqMmFP-qm0)fA|GKxiRDv+dz=IT(0
zS(_DtNZ@c22hIxxn?%G^L9}TYGKu6Ctana1;+Dme*Nsj>C>s;(w%{EsH^RRuqdSS1
z#mVSTq_rks)U%WRc5bc*xFCqn^RLs1-M8AGi>6@Vr}&96D9BB*x3$1W1)3$%jmEBj
zB`F4;Ju9xz-+6Q^1Z~5bqY|$R%(G7c#?k*Db&IP&@EXFx{um{-nIjpt#idGR#^MgG
zJ)X+|$_&B|!XNwI>Kw>|ob}-Ps{~~Jl*e!lGTtpLFTgU4^fgC9ol~K<5zGj1s+^hN
z7_Z<(j6gO4Oi_JU{n`z>G@7|ul6_uKX?IUCQrK2v?rlYd?F49Y?CP@xn)s4OVBtRr
zaYqq&*=ZKNy#RE@I~1!4$aQPha-Vp~-+*!lxU=Wh5kF>Ot{RQG^vY-4St6W<dEo30
zA?80sCWL6I`rG5f5|-w!ohWb!L9yOl<kq0Jw8TOB+k}*qBHbRs$GdkIC_JCENT}Hp
z>IM_%Cq>p>=^iq=amY-1)ZjQF&V*hw&F1q2xmVKpC#T87?gfJ@su<4?#uE3DqD8Yf
z4pj6vgp=nUHu3~xz8|0%YdfCuSf)@aX!OZR?X(9-xgFf6j#27M4;7_Qoe#5#u!zQL
zLfsUcWO^|2s!YVNzsFXBvsB9GUT~&B@7^A(t0=28A-aZR-LiHz&LN3Ch)~}aYF^VD
z8O&n%e=mWqO0gjrR70L*{X9WE<6&~R@i=MBW8mfqaWj!D`t`s|9A4(a{#YmW<4Qzk
z^pv}yMB0b-Xj?oWMARfSSc>}jv;dl;!NP1$`nfG00fPEnweHwae`kK+D~S3ELf$LX
z{DrZToucUPpkVVj7f=lThL0Bc6tYU2uLRe71m}FH7JA~=#{e>u@K~i5o;gndnF#6H
zpE&G7Oihe>hKTE+*&xu{8i5rCiF)CX9p8Qu+!TXvhy||U5C)4(Tc1b9V=txrfkt{y
zGJYb&t-#$iMND4mDWGCHU}u`F-PRZCmf&>C{-_53P@p?Cb;`zk;35H;&AF>dBFo(q
zFdm&vw_PheU3_EQgMgvitVGj;m%c9Q6v~3HfX?EupXW67C(EvnmG(EWsZNqf%=o*@
z$dz7)#d%6=su7B`8HYYg$~{CZL|#a)c*hI%0JAm`IY0&vjW|iLX;zn|4N)wFn01Gk
zPk&X?l##`xEfgQld0I{8{huR`$;{PL3YK+>AZp<;Bi8ts`#i}!G@~`e;6J)V*dz$w
z5w{3&EwZQ968oB<OmL<Zf!KQD*DpAme2!E1dt;Xuf%2v~!Y2BJ(r)wkD}n9tfk3T^
zoZk<4iN9$bk;b$*4Jw(-4|mq$7Ny-1Ew$1nKJ+q>c>z;m#E4^F0Wg)(HXx9p;;+Q@
zLR}Ikm1IqvAQPT#xx!mWxDH~*r<}QKIvg~((YRTdv!dRWPAP??dQ`B#jrBP<`LC8=
zX=O{6vb6DP@nO~_f>T%G2PMS4ZDOv2Un@Q;3lt2k+T&zlta@lBr^V??m~u=35t!BE
zJb_Sd#`LLix&DS!S<Mi4D4mBM{y`-jy{`kAGWo@LMGATU4bUvD>dntSRn6YN)|;+i
zl$|zn*YSc(niT$4j+HkG)9AGjljyDDclcyTYfvm%Vttt3q}!%--gedL<VSZ=$s8-l
z{X+z^3N8LE&}Ra9WJzy_+W?|W`XTdC*Kh~#&ao;aFb^?{J$K3RfhsWnv{#4c#g_jf
z?E%B#j8N};3dl>JO<n&jg#L{3%Jb3h6xoT`Bx?i#u5W6~>^X6Z(*C5*!Mf?V_TNPr
zJ^_7?{||tt?Ese}FDWT{O`8)7zsuj5y=8r|xY9JwPdL)g+3x|E;M_C9jC8ZD#BfsQ
zdLO}Uz4u*Ef<0DSgE2*b3DgPIkv%v%KQ7Q*%sSWB(+U?c$u{}`I4`^$L#2@7u#!-B
ztphtgtqch=rwsEx9DG8wQTS>CZWS^T4?=xxqKe%J6fU~j=Pdj~fvz{5X7Das>BF)C
zOgFt8jtIW0P!A1Z>SQCig%H>0N0M~&a=k#)kkj^9eDo1%$X#b5Fk;JpLedn}Z7mk{
z1fLKFnOSZ*!f=lLPylsENYGvRQlF95yqYpQuF~Ibude(=ngjn^K&^Ex5pB^W$TTFx
zA^w|5EnF9Bg6km2y1~q9#vx_-UInpaP>&VlUMNak{6uMYeyD;yM=gF|A{AWD(7><v
zc~PcCn_x}pOzv?>G&<(neY|2BAVsgzo>J-EpdmeKiIOfJ(hn}pj>pMcC)tiK0J`%-
z^>)^f!8l%!_;=RqSZRL$J%=F3=jd-!Kv%;Ox9HcT1&Rt}GKRDkI|=f#))IT?1P7LS
z?D0i@F}V<;`G+=@;TjNZ+Vo4MeX7#q$7^IznapT4{KJCa?g5%enM`pay0QdeE0hW-
zJ`B2BY*bBM8egq^uGGP&kryPjeq|wW^A8gZ#}KqK1H#zVd*tRHH!Msl@1S-kd{qcN
z93uimEU_pF=rwcJ<hW}w{&r8;Z#rYY#Yt098^U@eDoc=ZMYe>AJA~vO4Py_d3)lFt
z1QA)=txow!AS}&cn!^k_;AOvLKuc_j%(9UH*NzH`H^pKzA>$*43}y-9G@-e}(g=vV
zFBC>QrJZeyyObKA5h&H5$aY@_qLZ_DGVU$?O;Z{eyC-v=U6urJFXU&RUeSnq1(J*Q
zA$Mcfbzhka?`dKHXr?|Z(7ZA=-YbjfJxweWm-4Tbc5D`QR3i5emqYH}*ZK;`IP<2E
z4$&ou#Q_rOWtnG`q+cALSg225nw04QKgnrS-1iHiG!(1fT6(yxdl`UoQtL@bEehXS
zwkXYMPLiBl94-{f7qoTN;(4VqtMEJ{*ZQ9eF>Q3aX+e9T0Q3;x7Y`S*Y<{=I(iNeU
z6e}%9zB~|3MOEvK)xHX(*@<UBGygqj)|7Z&fBVbYFImigw?a`ky`{wv+$u<oB{~K%
z`pXQcRFbLw*H?4~6rU)n9#$%Ue%SikWmf_+3DX%8eqnB%Qle5TKnn?O33b<*>ph(0
zpke~ubzN_zHRNs;7Zc1v)pCy#nfhFAvVoys<5^n4?jO<06(?ZY${?O=c!i>HRT|37
zKqK>W{q2@Z>!mPb7p&s!=`jqpfIlkj3cIV=wPRDYDu~<5C39Bf+#t}|rp`zK%1-|p
z2rM?nd(OB@wyZ|d9pW}^v|ZsN#)Q$776jg(=f7vSOo;{705`YKio=xf=aShl7M7iH
zra(6wT?c2;dORY?AK>a^4AoF0o+-;}AxB?;jPRB~H<cqD&ss#RS$qO5RHEfjOc&@+
za9N(thX<DV*v7allG*4*0_i7vCL3P-X;M);*SOMy-z>xx<)=8~JwcxIupn*PXfl91
zi8m6)5IPgj5Xi_i>X`Sk&-Jb4G)qOJg4*g7rQHz;&&XVl3Gt_pU%4i$#4cY4GAGJ+
zko`fks3|n_9P?A!ZP;F*<_wWE@6g|TTOx?m1<$QbnqErq<hJ$Xbx8U{a+6V9%Qx07
zq2v#OyX1O6d0|b)XX)&@-1?wtYEF%N^tT6%_^e13Y?YgD0A}JKJYj%+#)f4k1yYi$
z$+V5iOoTJVac2sE*&K0fIMz_wwcts?rn=@-0C%|!!<rQ7C{7UQS;Rq}hlC&ylt{1}
zK8&~mDDf_#u7TYTYg4VvC#nX2r<#<&KNY~@(Vv1oEI+OMD$Jm*yfc0y&;y5?3b)Pi
zUm>oA<0)(Kn$rQGcU3<2n9$B3>E2;K&E2%pn#4RLefUiN_E=!$n_T~17vv8}WQ2y>
zwSZ)zR@zzBP8UFDQW(mbeC?4Tx1I&aS33!-%_91gk|qPjV7+?KlGeXC8`NWlb&GxS
z6K1J_>m_2>5!~$m%iG6&H`o8zq^rC&;hdmp9ihhwBEh0Jp=LhUgU@=-?KZH(YnfBM
zFR6|^rv?UaHmbxcf;>7}rX0F<fjNLC1djwDjmB~tgV05;kYzHx?j|Hn7xqflf~f*b
zX<S}<VuvyV4;&yZoRR@Fde*PJxhW;wLX?--pTj}z5t|hivpn&<QAGe1qxf#aWF`8x
zC^B%aB&_6>0$BBUM!OR8e}e=AJcszeU)SI6EDyjkhnulu3CXHg9v26fP<lJB(HBPx
z@<{0#0Y9N?{8AWSsaEjY_`MQ&fahtWO1C9{6zWNV1{Z7Ae)O5mSmc|)rX{f?<Ug{M
z5W0j!tKfODk<xA`tFmN==n>>mVZtzVIcUT$1k$-#EtBIO{mpb~)Q1Vo`ERA%;^eWf
zUGu-+Dypauw1UNvL-{#dx{B$urpJq0@wb^znqnSm@eSTTw=OEn7sJiCa9c3<SQB0=
z1GDXNp>CyK*TB!DrxCvsXfj}z$5vta=8uBSApL{5MBG}IZ4-+nWye1!fW`j3-gcDC
z6IEWFtot_#a&x4|(E<Uw?e_kJ4brM{wbEu%;&2nLcby#yfFmZsxPMRr%=Xc0th^&R
z80KzGjT?6&;SZBNUB`bY$W=0liO6^3&IN#>I23+IfDcGHqy=1{ZE4Y6`N70W=ktcH
zjVFMLQGi)F^LcUQ^he;WdM(cSHUPBacARamFYZ@bTS3e8XxWXwX)*7S^u>@8<~0S8
zVk<va2=i8QavRrLN_rGJ>&Uo!?@roX#LzN{O9y|)S!C(Q${uehZCbPHAQ~mL%rbL9
z%{&Cp*8PE3uE9OQ-Ai=M$&#f#5@`0stE;znfWX?r%9yLwV<~7YIoh5jG`EL4Jr?o!
ziy%{@6P58$+$F%Dmi|KXCB-rP>UWE>G82+Z?qorx#|RI?ur6LH#7%*1;3f9@7lpY(
z0z{|XCRvevYcFtrnZ1x(l$wA{Ue-#gyM1pEMxuqi{}cVqNbsi5%(#Ca63|_j^$h7k
z`xYhJDoyUJVgn&$fTQ)i)x!Ic%uJ3EsUkR?DacKQyl7fjI<6Du$(5op)wEu)#_V5|
zM#4&x{Feloz6vy)W#PY1K(S`fRS>l%)E{rf=u^RfmN<~&bY^G_mF8MX_;=o!kkx5C
zCdk#xZZ16&)D??-ugHjhR?<v-Q;?`Kn-B;ev=+Y;tb?U=AYN(%%FJG8d}C@mfLXL>
zm{<bqRf{z5APvRFN}B{a6{UkZjJ}g#4`Q+<kcon}yo8Uhq__sK5b~&%GQ0IME<$+K
z`xpt$+%jB6*<vJfOJ33m_fJ`4Sp0b}`%Z~Le~gJ~(Eh^cIF7+x@wgJM8beNnzns6L
zXo4a`=2dceq0Wa0IS<o|tIB+>l~!D6lZ<;oC;3b@Hb#s&t`T6W(1PTiGt54+Y*&$w
z$VAMh6$L;h2X;?;RMOPEESw?MBETt|kHq8}5N%0sy9{vw8T$z_A2iU-RtO6Ba5s>p
z5-Fassq58U0@PfjhhI!5X=K^piE`Hq=2prYn6oK&Aq~SGqps>$t2Ck5F_d|ksD-eT
z;Jm=ps+}^MZI4@&cYLbw*fS=YF{cmAq-@C2u0OsZ#GJ*ZJ{(&s%?ROmjtgIzozmQY
z6Rp3fl(Eo0h6?<kqEfbM1|wb*k}D->a;!P%O1o<qW^bc}2_}ONNyFO;^2o7(v{vJ?
z20-qR0%O3vg!~Z5F2ah-0>O|lr<1HVbBG_zxj67pBini<66!iwAvsUUWg~%}5i$}$
z<TSHRz>|`JzO^Zae8N3Ac>dzU-<s4f3HBFt^k;Cjp!KL7EMHpAImL|RWS3sO8;rA+
zbbAOQliw-7wrqNPRo0NmZv*kFP=A$`mvN?%=q45%2KMOTW1+2K_YsiXa|Qmcv4UVk
zD%9lcAm9S;!H)uHP|>Bu;5-thKd*JM#~@iXfjnN_mEr!lPHC;+2vx0P{(JhyQ)A78
z`P(d&EYed%0Aa39FPPLhQCu!)oE^~^%9qw4!Wqs~+-qH-EN(bUXBtR4c(qXXTldz3
ztZm}tR|Jo<2Mvyu4gqjGTJ;#7EN6!f1@lL7@6+l*$^OHOk_4qf&vvH(m!zqFl6uM!
zU{Gg(_hDk%ktE!mR%WRDcy<)zw!(KDpE-_HW4)t5Ez_j9n~rzO@-j<5xFN%MGI_;K
zpufvq7t@?zFIy_3yVX@`w)e-j0-OnBc<f79s_zj-<9OX;My!0a6V)-~+qac)@jhPC
zs>bjS3V{8JEB?0xxG#FlLu=oP#{e<X(%VXvyNA;Lq>T!h;};8aF-4V=Y<Zoaael|d
zy_UAfO@9dL7I19hLhvF1<|FpkdR(ux1ae+uL5@6@l)ExDwm!~TS!a^5;Yr7nc0;h+
zYoL}THnXtt$;#>^_g~qPCfpS6Ym+Oc9-m5JZVXSfb;MOF?QRVBSG8goazWj*=s_~1
z)$+^CDvr1f4uEA@>*;Xt7sTY&nM`8-2_#(=3slEYLrUybP6RZWnDPS(0}N+Q6Y7RE
zFvypM`Rjt*TjJ&tSde9O{*%EVRx5HzC6-Z&&oPpSIWg-LlKxz-1MYrXU}9V}hsT^s
z0@|lL0>1h?rKtdWZmFBU_#=|0WkrU=$l*yOnl9WeX;siy@!XWt3QT8eS=ktK*U~vV
zy@FxX&BA$=lZ~YPRNP<^<c4A9iFKxkZn9^duJ7dCh#i|#=1xNB1bX@OFwnkAn{<r-
zkgk$X5;RWZwBgDM`lYg}fy#iMtoX{0U6syw>eQJ^i~D9xpB1kw0dZ{V5+5k(&-7HA
zc&An43#S8#brtO{HdV@u-BTN_#c!3SGQGnRS1XZQ)P>=N-E{sb)SYW*4x!Hb<xfCZ
zRc9d=&pxC06-Qw07F+z(zs?|<%q{vm4~oKo>8P~TnPC13cH{)$O$I>Mo&{=JEA*|;
z8xz4MHs~x4jYy!^E2vBS{{ejKB+{fG#<w_aeqMAF>^5*Qi46)2_g8%_w~K`h_o&g>
z@MmD2cQ~kEbUIf+9#y?9(;gQIfm*$?AG+*p=VyQQW%d&SEG?z4>*V5nz5N`1VU!3-
zpvPdkb8;X(02Dv@IluZlQ|HWSA#UH4b6qv(I+zywvqJ<p%b4PX55{dm7*=S?RooL3
z3b4I5;wb@q5qSgSmiSmn)0|z3<9Jfjt#Cf5sn9`?t{&|^-x6jbqD0ZtAc^S`?BUU&
zNB4qN=XLoZMGw&Q+@`EEHB{zqy9sk)Y(_Zh$b@Bgfsh*=$6)%zKQln5wa?aKkqevy
zU{p5Hv5WwJk88Lj2FD~}=J-CudPxnNln`$cVqDiBjk2U-)O9Jo&z-_(fEFf%;=VZl
zmjDo1ax*7&8oukVK*Z==&)t;&p3WNtw_OMh@zD+;B|LKxDU*yPu8?IP_zkdoP1qka
zD%xxl=q}V*qei3=n2-^@aSF88hzC_Vj}6|Haw&^vg?aK0405aWFQwggZdH?RakGno
z^5((SWh1r`0LykO*x63MCE-eO0+m_IVC*8yl_=b(gpErD<#j-zfs=C$rt6>(V<xvp
z8(vyeq#2dW$Tt(_Por4Fo@ejN0H6}8RZa&7C`IFS0u^nKReGEfh%`F<w_O3|4<jz7
z>n)bygqptSzfgP(b`V7@I+Qv0o<osBc^)mu)1=)mIi4WIU*Yh>$pdD1<o5+eW18uN
z^olUrI&FPybmQnhkjgY6e213!hXlEIGB3chS%AOQR_Ds9J?6U-L<{q5+yoX@B2Spw
zAfla4@@HNRm|KGvZ8Ece;TkY1mJ4~(Q@<?0MBp}sIoYaL#jm**G>Z`Uyf6tdcjKcU
z33rvu3C>(>=^vL^AJ*;=95=q}it;Q8Y@ChiJV_FZ3U{NY#STii(VDzmo^jN>M4&s$
zGafqS%y&b9bdwKF$(Mw<Ab}<cdm$60<peU~v!_f+mtSYyNLm{?XIS)Hhu&1wf*Z3A
z7Pko@Ll5EA7^nUIMAD-me|H?T;;Vlypsps1L<Q#(K^_=Ja%?DmF2L-<ddrT<u8&Cb
zT3J*tO^QJ$-Aqo;Cme7GV|%4x0vRe+;^*bBwOUt4Eb|wBH66JBs1RM?W+DDk9}VVH
zX_`LxS75q<si~AdA1mqctnlg_It>X3&09fDF^<&)(8abd?hxotOr4%+_J)wmF+6HR
z$W2ThZYzprQwHYwl)n`qx1|`9X|+k9OS68UhT|K1DkH5UB(Su+9c4N7I@$Th(E?3*
zu_QAf+U@`{t$J$BdR(it$<AFEjP$ku6GNBPI_PRlAHTE6hiHvNy7pZFX1H1Ls1h!u
z(4(w~iK!308`vL1-!L5W-Amf-=|Rkv6f{YYNri!<>^621!ur(R)f)r)+Z}I*-DJcP
zpOzTPM{H#1{iF9470cR36@Pv|5cFvbX=RAn|L0`Vq4&qE2b^S2T65}@*g{Dyac%4=
z@ja#TvH^pk3yMnwnM^FCcoe=Mz+IDTLp<~#X+EcE!_<0LN#`Fz^EwojhX7nBl0cfu
zH9>hO>fJnr`*2Z)V0_4D-xL6eM=Dsg9ifB?g&nGVJS&fYjIXkK9x9p5b_;brofJ5=
ztp6y8Kb^EbtTs~xnQK{Jb#upIcc=uBsBz;YNLvgG^hfm64}nGHotRL%mz=zGg*sAx
zBk9}t#t#M2dPI!jI7x}j!6{fc8Gch@dizjDIuo%5f#y-sRpZGrA1YRzRIBltFf$y6
zC2*)V4j$8f3^;Rhy%(>;0ReeILy_v>tuH-MGzO;$DMqVt%#&b@A2h=%KK}9)Nf@F{
zy@`jEG6j(fG$CTn--|j>{zy~O9xn^?*X6I?6K@Id*KP17#$5RyfF7VeY>GK|uT>(f
z3({C`BE+mw)wSZ>GRLTcM;s&s>LM=;#}AayT7XWPHRSWMbdStpSNJDCd(VM*rJ8Cx
z;cm$Q+9=k_n@fb-CaI8?e}+6BI3f#}#ddtIz}SpAUXZUMEFb5336RZO1?f1~Ej(?=
z)7g-lO66I=h;?#T`x8NKgyN-eEMN0^5U(sPv8xg;p;s7nFnLyx4-US|9T(3((+_-0
z_jUC+<^@s^g(==cSo7syB$=0Q?hl}gM5H@}<`F_<Bai@sN&?MX2y(p6!GW3s+=RT+
z(%H`<fu>KR9~XG>LBiZ-#WiAak301pvj%Ex){Dbm0&ur6mtu{HH-+RZ+`ywx5#XwI
z;2~lnj(;x~nWTDd#%U!$m<9xlpOyfj3At`s;B|^*nqwfP2Y!w#p%hDoihnG~ALH<X
ztskRxpD@kf`Y5h@N|+_5w#=9o3%*G@({q4L-8KiKLNhtKc!o@d8;1xpjk&DCZc@VX
zp#t51ZY9}L;;=HC%}wMOaiqXJ5u_MAUTL#gqlxpfRE&y7f3k3YHy!cOiN$5Y+;N_d
zNrs~L3UZ_IU25n3X%_QX`z=t~lEQ){w%A#Sn~ad7+gjG4PJwO<BKJtF`8H|Tje{L4
z;O<Jf2t$s~oVxx8fo>M3HJFB`_Wi5CYy`~-;6h>JlK`2pHSQ5Y<qrLP*ndcwH<AHD
z9R4nV=|BU~ZRv?;$hz)(;O?Lnb|Phpp%d!1jnrfcs&y!Vu2Z&PSTU)r`-jH5D^5__
z^pg>SV!&S`2#(;X!{K;b2~V)jN_V`Xq(8z=ftj1c)wSOT%>2-k{eN49xn3l0xy8nA
z9{~B&Qbot5N>g<Yw(2MnZ&VUubfV<J2=GJysE)%Tw<rniSc2#ANjAn!KQ_Y5m#Vs}
z9@~Fngd*12MMH>tuYi~6S3WHYwQ(!P>qgHAF)a~2h;NZt<5{6@y)>fC?|4p-JK37#
z3r7Cc2^gwPV$XNQJHoWE%_0cmil32i1K77&wdV>j{UpK3;I$Eb0_iz-uS8rt_`gM#
z6eV0%UvZs4Xl1p0{8>rY+mNqE+&uptsR}}IA=~^zDcXR}6w~(wNSN*LzLN#X)k65*
z2tkyparzfY<=LhKBqylEWoHY|T!vdx-LHwBBh-{pR8sQCxq?iY6v0P(^!dWPx~7X<
z$STggu!K9S*zhuPx&-{4;5@BhXAJ&f-QNrJ_s6Z{5duG75EQ|22KK4l3-LEZfzvhT
z9#GP?B%g+gR>9td$MesKm-M%LDyI)rY`(63UAQU4vN(&lT5l`@Ld~d_WMTQXlI|h5
zT*yWoQTi7JgT`3<6Zc@Tq6Xd@hDl>LAuhrLOcr`?7sdo&l~8Q7=v<t%N3lD-%ed1L
z0PaB>>IS!N7lGtbm=q3qyf-eu&BbS+);}-y72=QK(agmk4LC_yW?_ZqLH2vIEXbjQ
z*Z<-(A>L(KQ5AkEdpB<R<>DJtThig?Ga)byFE>+guO&(Oi(K)^O1>E@d<Dz{hDwd6
zSJDCNK%t)Xh_nPSUYc~C0enzb1dUi}SwMHGt%*$9Ya~~O%5bDAckur~abB0NJ)DwC
zyGjX|?AcIhWmV}4be`w#Kv^dVHB}JSWbG0=mfvpW1r2RmLfko&ivxs!YQ%HG{0){-
zo#?6UvH*172z}BMYc59`=A1rrPF$_Op;r$_-_Ce)c~S+7&7LzSR$sw+XH0L2E%dh;
zlPA}Zf0--N6b=ga@YuL!+)?k@mH5G}fj0Ht5w#|P?i*$;=?RJFl+O^W=l~BnM1tHo
zj*6JMaGU(1vKbQ_NcxSpgt^6)R*be9vGB@8Wwaw%vAre81xL8rNhDrr6)+P8m3TdN
zDAPnjXpM8r^t${Pzfjt<Aenu|Vyn6W4_j~CqcoF|^;ycDca((4Evx`vT+Lb81p4aT
z^|)GTH=3AAy3N^tbr6%bw2pv1+J)yaCLW58(y{9rMc#DL2p7ZI?QzMP;HDxnBGD83
z4Y@ehIvol}{6829hGcSfnTlpM#(r-I*%~(9VG^i^4@D!#JWZcn1v)EnXi>B?hSk_p
zs6WKUi0Lb;kmJhY@-T!`I8>b?(2YdR!zsEqmYNI%rDhX(YgPU2pRsvPO1-uc5_fU0
z9zPNUsavZkUE1RbCH?Vk-i1g3c{g7RjCD_OCZk7*Ohw&vvZ@m9yTDLlXfh7QaqEJ)
zZ&)36N@)vSS6L6#qXh-(DBG(-a&LK*oaL?=^MtwO$;b#%Cqd4J8XF}I<_`%o8)0w6
zN&CDFC}*Hw#u~BDh5#_8A{I&>IYVh?C{j1u;6wRWy)mZ7J*u>e*Rg3Ij18v%xE&aq
zCR(gJ6-ax13(u#$tAq=*bxG-X*0cg3<P-iSPYG}h4W7@CRJ!$aAaeq4VQ_+`-u41b
z7jEe)=woXof~io2xZ(*hMcIaBv8fTCkZ+JphDKbZl5SN85w9At=?nuHViNrc(c&Zl
znKQ6fl*RpPIS(?84Y^mGS=55ZH<S8_Sp^{Pka~P7!0n;II=^ebziv*Mt=U@ZjmaAq
z*$_W*T*Lmh8P5sL2Ys!skXI7zs+$y<Io5ONO^1qvIy0vO-{SMB1iKq76yoq$Q{_!u
z8kE^{vOwnp#7dwM8wxTzVNZ#w2^nc4VO}@TlgL@Sk*ostD2ul_NNIdWkV#zaRbb;M
z1(+5|f@4o%-CAW+(7ZOW7Q<o}2!SC9N|9F1-zu#wwkGfReK#ZJYI#^0`;1`$Oitce
z!6h|z-Q1;F`$iSQWsh%=X1344{r>>{P1YI?e`qxiRnnj4@=Zp<D}1X!`mo;MJ;M)#
zxY69Y$i0XAYojJMi#MM^4*<ndW?IezUcBzw#KM)rw4BHiyg<G9mZVHic4HPtMCE0K
zne1I$siKOAwFOZ~Z}RoW#!68or>+6`Cl1JIo$Q;nk@z^LlcyBKTX-w-XR5Q@VU;3)
z>eBf<zNZw`QG99(fn)AgIuH8*x*fKL<F^KaNGWOw&kAotG82hikhxP4LIzgHDls?T
z+?J%fh0>Y3G1mO;wgaOTEJ4lmV#oFZOvV%s#Pi@Wp{^StNGg|SOJqo52q8PRCy&U%
zF`MV|b|5vr^|K%mCeISgL;9guF)%FExh8>bTD4!NAqez{GP)(5b+oz63cm7aSzMlZ
z$rU%A5NP6X{!Vf0*WIxw&3q;tOHHvHTZCqIV9?kgB_74$g6SRsa<Yl-O+p|Y<nOKy
z$9_AJ^k5OthQ;*RodMinbh)}_s>bWW+zLVyVzDzCp9wPE(uFe5<L*)v?8h%>RL>B#
z3NuMK@vwI2g83|gE{eKMK4SmM2-K0wD=c%tw~NY{<Y~8$FYE@!nvR*ya6G64m9YF|
ztN2BC2Y~`?ya+scgDAavySiZ5ZG=q{y?^YebUA!-gGD96+J2RO$RmX`l#V|WFkCf=
z7syJUFgG6wvO!y}Q^~xVv690jY}j{}AnPN<*Vc89KruGk2{Zful+2W<bTW{!@g5*~
zIB>ja692l$(!&`TK3VfSAdo>KSsbl|8;!1cfb%@JJ9EK2T0E1^IzALv3i5|wBQCCD
z>YgC}5c>u)>U04v#e&{cl%!?%D)Mxun8m*qKs9pcMHS3J>G_PHGU8H8b_p-<?L2fE
z&9pX)-S+{ZjW#Iqv^z(bzlI`>n+~1?5v0noZ$Wh8RCGly2&)q67oyVpS8}d_df^SR
zud62)*Kr&i5s!8d=0RqiXCg?hk)y#s^@FD#`gI)@H03wiuRys~PbSq2)(?bwq3Wtv
z=0)>B(rz=7rHqZ^3*RdsnfzB#M14(=nL$yh(8Kb9H7bCnHg0WPVg8?h%rZkwoc8qw
z-S=AoJ%=F*CaV_P6YLF-!+yHh?vV4>5z6jC9n<@tZE=(kbHltEk`^LfoG#39Pt$=U
zxx>JsaQtR%{Nz><%Zk<F3++Yus$=dA#(*%Wpb!s=opZU;Y-Y1&OpdGdw<&@qzajB^
zn<|j&#QYCnPDcTB4stygTXh1s*<3wx!4r4ZKp+NTWfgvf8H5RQZ#UpLwH5q>S8fGm
zqB&q8;5>ouGwLNmLvlazVTo}D9!&?LxjjW;MqU?}nq2~EOA0QMPU0W+f-plxX_~fp
zKS_E!gLC7>`rBXPgvu<UclQW$cd>oNS>c+40Ngvgtq8gm`wxP+)huP)&&gaMfq75Y
z&Wt*nm9e6%?sJ7_m-QT9yP)yzDIzOfKDo@+&(o8N&G!dEc}uD4eN>sj#p^=N!Ca(Z
zan~s7M<JUWa#j=M4zwYNjl~QB^tc6K<&XN?osq!F&fGEt;u?FawbuAJr?LG<4@<0_
zQGSPf9At3Aj05M9Was2?#d`8ula%>elA8jmXfxk!4A{Nqq=FUi|3F>!_D&uBQ!e*V
z+rco(zyC!!sBjER8|?0?(Rt+%iV?6*;d3W^XAr+phWE%4V*avU_r^8{7p-B9VQcP>
zy9JpHY)^^-_J>12@~B{y!kX3=R}1qvBb#&3kgzcIP(WHTaq2bW)WblbqGBN+i^Tmo
z%@Wq5P!mfX&hI9aw5-x2%j&XU@Hj8jdy-4wCBi&@C^{H{+rAIL_COSN?pco@;mv-m
z*%`+u>17ccH0?q8G_d`V{NN8VC{=CV5IGX;k?CSFS16JZ<#ZclO2FMue;g^y*_c*I
zU_POQxWh?yNT-w#(}xADH!c)p!Zx^9gSGD!V*cu?jNxH?R8ciW`lSolLJ)nbV13O2
zY&RuM_Q84+L@V1q0>^a=@LHR8d8$7{ILtkJ>g0%{`P(0_H+YJUaTemfb)=&bB~JSx
z7+>Rw7+w0Fr=-cPAkFQu!Lb048>RB>Sm-zsCWO{QXc0|8>G$a~;t?fWdQ3Ek_XRK@
zl5x;yNsWTg!kpOZ_@btPD%*a2D7F@4rq)_JIHWx-wh`(o`*?L38#e)TB6UDg5F6=N
zPA~?4rHg7x#<9Z2d!l>llLeR_xQ|yD*B_q<;_f8N5k7Z%33XsCq)W^51kuB`v1U)~
zdQwqqD|4<M#|m(*lV&D$<>Dt7wIaTA`oo%4kTWH-(Q3Re1U^7mN!@(<)FKC4F|kr(
z&S@YfM~#g<yH;&2)J?|Ev{kERf?PRER44mvLOjlWgnmt)=i8`YUYn}2W3!yHZCM{H
z3m2qav7I0@Tqo|qcCJVu3tjRwj)TimN$rU*&@qDCigsMJ36Ord5Dx%Xmzb43RF>j7
zXKrx2;v+$DUoS@|y0Yu(MJ0r>RaoGko>3s^R+t~L*iS(c!%dkHt0-Y2sriH4>7mcM
z<;()L`yI>kKwNiLfif;oJ?Z+l{~4%ztJ@pvxf5t&Rp%krjVJ{CP$ZZ^L-b?GLhw^r
zXTyC0`QJ1>K(I;CO2kQAKx#s~F3J9zt!p%e>5pOk;Ne6lMF45Um>@TZ_gs1NS)0RE
zxk?!WvaWHd|GE(Od<-kp{I=7gXBUOJi0GHWVtWh;G&`_urSL~_Uy12rCq-s><6QE&
z-AH<z<>H?6K(d&L;Stk;HBvF_H~a-ZFm9tLfNArQ=a(h91z@sWA%ww49bDll>uM}_
z0f4)uU}&b)h63F(-7%(TD|Qp+6>@a6R*ONU%|<mEg{k5%iwqd1_r%*uyAoVj*d&pU
z7r7A3bk!vR_x^F-MMXXMaMPk4eglx_4G*56Tew_^n4Cb-)p+V+Qd&z91h@Mwf4lY4
zim`NalH9)p<<G@TV5BC1Se2P8;sE9Jr;^ab0lzGp+;=!sp@S5dTa6hu`pc*ba_97B
z9fp_n7!>AFfwH{mim=N$1Y1e{;Lr5o2+d@<U_y=B50QcI5kx1`dt32FS%im0F}H|?
zFX4B0wF94FIV?5fj{+GK6qd;l=7>v4yQM?<+VvDc{y2Oh)6Fx=ObWD(A~LQL1mEEK
zuH(hy6Uy-J3YwX?%pXz=J-JD@Tu$2k?_@GmbOPI7Snej3qLyCm_aNpG4##MuzVQcu
zye^<%=3O%hQ+-0m3CH&lrnMdQ`iD$-!ig)EPs-~L<GYS@LKuXE<!mxSLk-<fmClAj
zPxcvuahR~2O(Db(&3+_^e&b()xnEpR=0F9;Qq+i-1i4cj5Qcll;+88xJQ0Y6QIA!w
zDyo&+88-1TCdl2C5}jDfa&h~sP^dwmBt39BN}S@NrmjkNcXV7$+KooY<x&yaTp^6F
zwWf#Sn(?^OEVCV0t;E1JB(h2pjg%~1A$&7Fxfa|##VtQw9L>27h@Q3%LPk!@N@iMM
z{3sb^hwFj7YVwe`4LcZvc{c!?@36%qmaI1RBLq4xW~&Y_g82MDu+&hn20L|vT?L1R
z^vq<=>lUC3mFWkug{jqK>%d4zG9uqPscZlWndBAFh_lPAjUF|sO;(wYmRXsTxW9=l
zSGlq105KdI1WbY#P7ugg@cJR4N#TSOZvyZ*>q-sEUr-_)+#q6Ov*4eKJlqP;oYNA=
z3o(Ne?F$C~_MZXVRZRJ0BgOi)r$Bd=NI_hu6Ndg~Ack>(S7T%7FU9Y~H-V}9<Bx(^
z4bpJMihm`=;2}rxVq|nG>H2!4$Rb4c&je+D$80Gn6wfZRDLfc8$FEC>!IZ~STr9{f
zW<|~qo2+pQp!<a*BlZK|6kzgn^F7#QD<R(RXT}jqxJ9x?k^p*j3Gp6WD)RMPi%QTW
z>!x*XiQux&Y8fvG@t38QtH;8(6@>;W^F*6Nh45t}XxHP%xBF*=1P%a_OwScU4bqFi
zKWsXGbCz}<i)AwdSyo8?2%!uHV$K}^<8zu?SGtR|iIU=PvSIK@T?tBV<}k?gxJIC7
z0!MRnMO@ji0wmbu!Is1%#8EC`dA4Hihl0$z?X{}H#N%=FRD1#?4^26GU8h_g0rVMq
z>EeDNUN)18t0Gfzod_4Xn;-n~N`s499%+chgn4dN+Pf8EQl{X4x(8HyDTnTd^*1a>
zxcRm*OmIp0bFGL*u<Xb8fusSSF+JA4p9J03^Tk?U@?1SzFoXr*B`^fZwf+x4e2lua
zKmMtd`FPT_8PWa#N!2(TSEz0!7y<@Tt4xe3O;tLGVt+Cmr#}eb9<)|jTO_Y>DJRrK
z<rvd%6$CW+AG|Tt&SPll1~lQO4Lc3aUF3aCb!6^M>b~~)hahL`;o>%(E#E3Jte2P<
zgn;^3n8{RSVS(?z{1Av)hpmVx`ML;h73eQe8^q$SQIJbwDvXvRt`cId?BrgvD^`CP
z0Q#U)>5VOwaEnG2f<Qa@zl6Ek0Y%zn9Xslg0vdPS{IMXHl#2-VMP2b|nGMB4TBXl<
zv?$5B6;G1Es67s3l5@xx5oCgXuu#TS&I8R@?#ZHd^p3s5k`6u*mZwkhJ=B}yp9-Wm
zwPv*|=6{NWTdvncwN9@r$j#K<JIAB;*hUx?DVjLKeQH7+@iZWuf+9G%6h5Q0DT&%d
zm!5O}?i{R_qHWx-v~zG}E?bx-{{fPxEt{Ve_ejHn-3IYpI!zq*ESPv&%Vj$TM78Gt
zOckuexiL-J=1I>NHKy5^mzo5R6JK7q^lGNEnmkfhNieU(mI7TX_dP?|;(BYL+O|;|
z!vs&iKvI(gp^z)UxcEiVCL=LWhmcTW@=HZM<A)zqI2#Ey1-Nt|;8Yy(GDz9gWGDHb
zSHMgat*5?Xs>bA3i>w|6`mp9}6Dq{oq!N1w$sItp(-VSd4|aRKK%tWySL8`E0cDx4
zp@Wc!`#&f_Oj#b`=!A~Xok1)cdH6?_x;L`ELLIV~5cfv*PTZGNV_#t=Gg49AdW!=y
zD02>!U>>_G%KB1x4<6}$gR)+!M(f?NypsMP-c%fv;+sO;D&2re`q-+32Dp`wLBYyz
z7P$!ANYmS5?-C;FE30Q`{Prz!u?mmyrH}M?<`<l?IehlT8E=E-g+Ue{@jC&|#Q6vo
z$5`@aVQ!+I&xy7E1u`xjj&k-#NuWi4ws2RcaB73G`M&{}&~>8PHRIb#d0Rp2i&`6T
zDBX@8`JbXb83Ckw5H&yHCIyeoHWd2s7-cmhbr9DwF#`hB34-&wfKDU1Z^y|3={;79
zaj*U!=ONDR*weC^{-02Hmo<&+UaYU)73Pm}^5M>*KjwQENZUyZ5Bb!UfE+0Gb^pg$
z94f>maBxd+4lMLu39*ea<Kyi2flM3(I1cpplz^@gi5>D0IcV<;Uch)tf4f7Bx44ex
zF8VQ`+A)h9mngxE)l-LEu<qnfNYfWQqHyHW_*dgvVIC-*<5K84or@A~DsugdhQR`H
zf=!O}NS1t|?o<`-9KtHWNeAPfCB&pjkEABZ&Bf9)-Sk%D4dqK@n>`(K*rJ|#AK~5+
zJeC&3@1wQup4eUqx}%k~+Fgl#l!h1s0|RlP{>~1xHX9wpQC_zMi+si}=1p!A_1f;o
zLQG~%XYdpm_`-L_JCKYukR4tsA(3eimxlWmAeZ?=@)uG5bi4VEP<Ly9PU*g`5&srw
ziZsv`i}61!^T|KX7U>@eG&QAyN!s9rzF1(oU0~DejIRsxr}2lxP9(+6-%;o|MbT8t
zqb2@Yf^uHq6vPqxiiKQ0DZuHEWxu|KL20Em(<zSOupn0{X;OiD8gYt1Q&5kMbH>|t
zQI~v<m!uk9NujjlVnFVS-t$1<X~qYOJDW6ADOO}NHd>;{R&8jXM+8b(o*d~-;<1>b
zY`GmgNN7*w1Ues1Z;8;GDxX<mjzSghoYP#w;UJ9_Ls<_tICWns94Sz8A?9;kaaJyZ
zl&zy&Tv--LHbF_)dHhQhVM)ecjWLN|f0?A0sk*GiErT*^ZeNlgU@bOak-0ZET?)V*
zfpnZHC4N1&G+-v^Pz5u**kBo#gP=XQl~&?F0q!fOHUzR-oG&P?l&p;LVwoX@=frgC
zpk>M6AwU_R9T1yMVXkKg!4(dj2ua>WIE_Y`%f4$}@hd@QIb19oNRzsU|4gX6!!lg2
z4v#Ry-UvVs0zqp>V%6nHGih79Q9P`*d{GY~&A5&8ZbE%Q%aIM+3I!at*8BXg@{?yL
zfm<NqMk@fAS_3=?z;w7q$hfG0bzwgi8=R3gToIUdjB*nwgQg!S&7k$@TB0{DQ_}UI
z=;m@RZV*C4XKqXgsy$aGVd^1hq!+g0T47!sSxb6Q%5dCZ{1ZN&7l~MIGJjTHQ=y|a
z(i$%*od+3jZHO1EfQ<7mG8UY0g&=3bu^x}&Sbo)_nt9j>#x6>`P`W>4w9XRbLd-0!
z4;Kru#8Abo?VPWXgj0H|IDo`^N|~S|sHYln+iC!=vadoYDQJAf>OhQZI`C-a)E01q
zKo>*@j*G<20&<shq_4&@Yk;`ZOz@%jjnWk1CRvg_4(<YS5t%7)yF3ZREX4f7=3X8|
zzY~~86tyS%uZo-j=AR5~`Xe*tFqpI~;m$sS3GDcSmS7JH+7FJo5NiQdVN4`P-0d`G
z0`LsX5Y37&Dx2Az@a3|3l8SRdp*f@6IM8A&RMwXsfoAT<BEq~JaAj7j+e?|XWZ5!v
zAyTed)+y1Gg%WGn;^PbeVj;V2Z=6<UB*MII65^b-$OttdJ~tI?UeUTXh&Iq!Jk@ZX
z5@v(`PL7sJy6dvQO%XJfS|{fh%g<KAgTo~WPpZX@LOh<mb*!TgS+C5IF4y7?A>)KX
z`s7>$p)Ot@)C1M2Ye#Hw(*4RM!p%bUF^N-m2ypT8_#V#-bwhf(nkXmY+Z%wm?Rr~#
zNO81x6Xvpg$cBi4a>Lp~Xy#@-b7iHxpCEr#@fx90nTK-)lOL0g3@PX+i?Z0_PC;&k
zR8cq_KUkJ)t#M4oV&|PQ6UPQ!bj;q6Vs5IQM&%q%&~ag|3Q11k+8-*B<SKyA|3>6-
z!;|I>&1kP6CN76C6hSAJ8758hJ>s!bif=)Cy=>GLhff7#%n1C=!qtekm4+q8^eWc6
z)BGbF3wq{xF>N~OJo`|>rKiSY#TkI^0hu5O+Od^{nTD9<u;x8oW+F_tlo0*#tC?i-
zh+rp)G7y#HfKhi0hp?nmx=<Mnv2C#8G~>^L{80%i>4o)&g_&%+fnxD`RtU`)K;e;=
z-c?%wEa>STPsehd)mRiZnZ++|40rEXfTb$87YcRQt`LzcaqMi;d1Md?Fc8^i<Dy(r
zVKJ{5!MU4)y2U6i24wpE>1IHtosN^0WWSU^FGWmI4n*v`@M?nn861{7I(TAn^P<9Z
zc?j!nD99Be5pr3m;rm7jrVEnfO?O*aShj8QvHXW1(+?R@RsyKWgwa{7t@Ye<?7Kzr
zO&khk2pvZYB8%=Y*>K~RO2aJ};$kcD4H9`wuu0`po$<?Wx&RzRq?dVd`L{?znB+7B
z-I-qh-V!KJ%p?IL`{2{CRZ*E@I&xpiD_PqVIXPA6DKKtFjG7L^#MtAog7JOInhDE!
znY-0w(zLH?80S@3LR-rBgt<Hp5_$Kk3G@Im-#I0>M_rHyrIVeLqsvJ`ykd4X2jhZm
ziws07l9m4a+Z70g!GtD=C4^|jojrX{RF&{7K@i6D_-O&mU{pp37;h+L!eB*?C1qmN
zIXi&5W!><M^iQV?a?9A!^_k!ASY+a~!IC-@+wKhHZgPTC>_2$ux?M^Tzc{uyzMH_z
z8<nv!32zfK_6_&2dUSK2IiWQ1wb^w#QKlr&?He7G16kk13h`taOPkU4`m(cO^WVlj
zQ+y)GW2VbG*dUo2?!0S}lc~>!U5RfC8ed*ooBc6Y0E9zL#bsEJQmzO$M3il*aWN{`
z{o-m|uJ!deQ<$5<70aj=jc<G#2x>A7l48Fa9YUOGpaL^+COSQ5g0GR_Ww!jL5HrIh
z9?{usch{mfeRYf<ik*beXk<sRX<O_fV7$caSZ5EH&|<5%C+o5GcR-kJAbPqhPM+%;
zNLP8iZZKBZv&b-1!>ao|rSp;?Ef|`&W~}mEK(kSLTuW~WVErtVx@YAWDVr#r34xg`
zn*Z2G5be`ripUvFC0#jtR}GhW7P@Q8VjSdJWtwtpE~XbW$?>d`rh2+?lN#}ay}%eQ
z>56OdoKo&4?uCSv*t;l!;h)yTfw)ANJ3-`XRNrYXY_Jcg`GD{o%;lDS0T?@U#VL+H
zF92jjRF#bx4^);2Rdf7_rT23wsWfdv*pd-S$-#(q{s<wa5BD&LUOh3gKadHM^yL(7
z#8(dhglO=o?gI8x(#3qR7>tDv1VY{`B^X^6vi|O$f8e8%q@^|(!a#{o5BkCUED~~t
zNY{%AHf0sGQbEPzL}j%orI$!$yMW5H9_I>oUUdDa(+ER*k+96xiao)Qzga@OY^BX2
zJ}FBof=ue<*S<$duVS;~7fN^#*!ytjS*i_y(M9xvxEr)PFTr@G#NJ9|F4G+bLjSoP
z&Q0Kb8gOe)wc}dWjD0%!%@pP`Nq6qts{Tiy>%j6)i(<Sd$i1TyNC)LV5@w=dz*|H_
zTeu6@t?k4imevZ$RLHlvs|!lq<GyH8`8&&xq|<1O|K2ilX6)Wwl!v>RalAGx$jwG>
zKwG0r*+&GL%iAk>&_9;bNY;r?|54fvL$!c&>BT(-(9IUzjQRTjXlZGn5pN5E-?Ivx
zCE=ca5ZcbAN;6(o!rV{@CtA&`1K@5TD@0E>ZRjy*LbTx}sSUTI%%_tg7O@P<1P~d3
zg<BCMI+fF{#}KY<Ai7GRr-sv6^b7DN)n8{<sm7>~yfCq_OU9xDhYNMvuz_Knf$8G+
zWmfJSdfQvm<C2>k<c}u%cf4GpOMv-6T9rD2R=g+7-(~#J3&#0_#TRtz!W?0lI(rn*
zjgcgl)LaC+od+sTXqnX#9ZHy*=?;`>l%khBE!>3DYiN3XG(l!QmQI!#IYMOEIf+21
z=)3WE6?RTN)|ag4VDeYXIy-CjI$F9(PM;GitsIfmn`FAVx_%g62T)#Ug!dL^K^I%U
zsysflxpUy?)GzR%m{pNmRv05~dN+m%yvOa;VEj#>hm%EQgeMLTE&yDEdOPB~0#Ki>
zzWVC%>qGe4tj+Wrju(_R$ysydoO0Kp&OtB@Ss_K+VWhPIu;kX__3x9)WI;D6tI=;B
z0Yn2DO#%x@vG!d7Ccegp(e<$_;r@2B4|=x#Z@x5+`r={2O(a=1;HWbkX9#mU8%=J4
zaC6KEM`lh{x~e|qNb;CGJnz&?XiQpnrodboO-T~c@kbR{a*)AxFjhGlj9ze4M#Nf{
z=Pfhv96r7nu`@#jjFW0a4Sh|BJA}RWKy3YkB2BEYTHL2Jq~jV5L2u5nq|CO+K5h6(
zN|N72s2k1o+YlBi$m)+CS7b$X0Q05DcN0)fDcUhOM>wx5NG3qZnEDA-;9l!yg4l9x
z@nng$VZWV(>}P~|I63K|;cUj+C4>=8IxjDCe346r7jo6SQji9nz?KN2T`_h7Kpr5h
zxrirurVzGQH02#gzSR?50(uT$^E(uW3vdZ^lX9=9o>XM%#@e|n?mW53(owBSyT08i
zMHWo2Ik_KmY60+?8kgmd{-`L^nv7&)$<siXjmerbNm>$AIL_>&AKZIbMHUcIFCmVc
zm>l+C@9JEb^XU+(yZo6y2J)P19qvI{bgcjng6{kXC(m|uhd>X4?B6ha>4`VXj5;fk
z5k5NhbTUE-g%Os`d}rJt#P#VeL)JrteHEBb#JG$kxr+4=ChGhNzgRpS8*Yu`lytuk
ziFkyGhsp$*`BGSKA5a!O3)G*Xb-E*lD$oC{C=K(q*RUy0kXJsr0XG!T2|a9rU7UD4
zRH-A$3o7njbN`|1?5Z(qfr&SZlTeo=qhv~uvNK0G3W_}Cx2s%T$%twKG+0AF!S&X^
zl`GFgtio}1S@LWU&&1BkXiwa)v|HAP*;hI>$MXU`Y8Y~HW53}!0H%{{=yl7pry#c;
zyL825ti+n<7L`#IGeR*D*ZW10vB5ftYN-+RUltf9Rhnc3;uB`VQy4*0R)<FY8TMpH
zhDtfdsi_l{;)pcMDm6XZdG<O^5$5jm2}P|s{8u3CeIp9y`@9nV8Ev2*{G0FBAaoqX
z9vAw^{a^k~kp)*kZRIar4C0Tonkv)?hnO$_*5#P8OdB@nqb>pP%8s%_LAE8^H{uJI
z7C5gu6E;3wBUX)@Z_qxAm`NI*U)c?Fe3pZBT5%KXHo&zg&!qiWP<c(i6eZnv=^-?a
z9Gjkkge_mTL1Hr}nuZJ&<Ez3=>V8@7%JO{mGA|8CXUAmOx~Fh<%GS}5*h_!=6X}`8
zj!JZu5KaX6iZ<d<K_<5LN5tY2g)kP}3=PEJ%M!RA>v$P2l#uj{dcbI(%P7M1MQM*s
zU0kB1sjPr9LtQAiv?yFF+$~~^kocQ%T?ujL!@UB29+wxnn7*wlxY6%`+(?YC(PqWl
zLg?sB;@n?!Me*xE+S238zXvhR$t>NA2)VmZ6LQkbd=&k;FfTll2q^N7l9}~%bRCMg
zvZ#uKJPRYfDD_xdsHbap9pSg*D$?#*b&RVY)Tn0)avOL$Q2I_CC=%*6pn#W&Q^S}G
zb8$j^foA-ytgi!KEX}xc1-WTFFGq~_IA6#(yFtC=HF#lJl7n@MAN6ZN=2qVqh`XC{
zS(%ejVspUuuB?&A0eAzILw0;x=0ow0y)=e@*A#8(uGHj{-Xp|I6LyRYY*T<c#T6e>
z2>aq5L0Q_UR9a*0Ye}<m_SX=zW6$eIxqqxZgvyN_uLp7eaA3n8SFEtr4S@a-3kR#k
zivp<CE~hL6O7r1l3((P2na6Jg@=#!eghndO*6BBax_WHK5uLSnB+#uG7-qw+7|A2X
z!4PFu(-_|%AQ4B2q#l@2)~4H2?HhgZO@Xehf;kb)y_*n|B{_f+uV8OMu%NEI(S#mT
zmTA-w6zg%15VrtriET5V5=NJ3QDQBdFVFoGsJm31m+X=li7QG_dlXtKwu$Ejx@D{%
zuzIY0GYE9Vl~WsbJoQ)7Gz$g40ueWEC21yNM@>;$B;h}J8*t`9OfdaK*FysH^2mCH
z(*{;)yss<RLx*!5$I`cN2XG?@a*IwOZuuLC$-`8@myd^01mzLIX&ytI&+Y{A;e<!h
zncMu24V~u_?&1eC*mT{oucWlco%xUb8eW|Jan3!UCMRkozjWWY7YtTxS)YX=UQ;T!
zi{%~3e82mOnpnA#CKT~uM5xy}TAOduVabHM|C(4BV4^{5H|)+a5V*+J4Y@62T~)gT
zr_#>GdRrkJZp_n%Sd#1f7w0QM>-0%U|ELx22hq@RL}Y9*J9Q4WJ&bA*i{pV3L3WXY
z)_Wz=ULzn!HP(25Qh7c!kj@dE+hVf7+?Xz|ve-jsmynE75C>wl2gxN~LEx;8#yd*7
z{~TM9@wr)nTwi+(czhaiOl5`Pm@*NSF3^)Rp&kPK-*MY_K9j1O=m~h7G$&$xZRz5%
ze<H?t5PcM;ChO?SuC28yBZ#<INFIyfb|mp;yjW%$sU_Q>4|Aq=<}Q{sI=Wx~cv+5!
z%L?LfmmqhGhvJ8-DQ;rScmgy}T12i^?Gi1*GGFl^EBu}ia}dH6<Cj;HYn9W@L#fO2
zAbqj)Q$S|!CK~*>Lur2?U$PST<!Lb37@Z+gc5S6xf)i|OT<{OlZYat!&g3$Lj(h(J
z>QD9PiWv>}eS%yyF7h-9=DlB-TZ*ckb11l+q8};qiWwyd#q+{o2hRAi1l{Nvl5WRn
zKaXW>qcp8V6s|Ype5L#`PFcJJK&2}_3uZ!bH&2YWc=S0Cw-Unu4jqJ?ejbRg<RERE
zyhzgQn+$?d+_TpO=52{GT`~9NA{WPQ+$XdX9{viTKR1Td&h^SsLh`bPYkgnZ2)BN<
zsEk**t7CDY05^*BQHNfQ*yJ@J_qEZ>RQRFNuBfNhj($N$`tQF1NXti)VOkvf7D+df
z#eywo`+pUcXupj^mCn3G_>?3_w7gSbc$-R0hQEaQV|p9Kp|EF_pxg<#d0{jYm<i04
zCS^BOwv2F$8mq_V<rlh|0~~u;4t6dvs*-x85xWXA6^4*>w9@TUR>pM&*Xi+~5LU2*
z^rqB(|0V$gP%YG$?>$mb!>W=v@O_eQ2<9-%^H}Z!5NL||H!7ZHtfMqJ<Oj)p4voJ^
zNTwiJdScE;F2V}f!$Yglp|mN}K}<h%QL=p#=r2_&#Q(=!Qb?W)Qa&U9y(h@DZ{k^}
zSBT=HGMip!X@xK=e_YgsMp(~^ts}&rR3?XN*+5`kE4Y@$$U}?Kj%8s?I;7J(y+oM*
zy5&Brtd0OHqCE5DOP^3(X23d&B-GqlkQs1<C^d0dnFTSJQ#ZcFpBAM!DWy0AhkOPE
z0dcrwny?tIG2h%IU(Z0kKJ^RyZR*x|q`ogM6To01Hg?8SN|-<hf_VP3ru<?40&5bk
zveHkD33JzVj?A<6$}(p^4?3k&`2-A!f{~)lSC#papodIv%)RH7!4f?bw<w!;9!}ai
zm?itF`4<58NVhc+;5*}TAzsedGtnFBxgt3C0pT7sPG4E(d{>}rO^drD@W$d_ENkU)
zA_nO%LLf}Lj6lD;kP}l|=1h$%l`sv{t#N!l5l|$K&P9qG$RylKKPN!TWFOHW;&Y{#
zC8%;m;e8h+nWs9k1ZQxnn74Qd@_4haG!y8?x8om>Oi0G_df2K1+~*H)nKGgLP^H>4
ziXl<mplK&(YQ(k+^qFxYCks&`7<R=fO8~oz!@bh0P7>fxD8!Sj&*JxA0&{IS9_{H%
z7R7q%?YO$0E5OB&+A7Ffl{jvxqL^YnFfYfJaWMjUYK^SL^2-8vMANAh1HvSje@pl{
zrD{YB=3gYxT`2UEXkjMg<(N%xxeGU#8C<px!nVwnvfWDtU@v9#w-Pv>F+0+1TVe$L
z??=ev3{bX+aK3c0iB!6mz)WUj79O)Zutapt)Y})Amz5z=V57*AwQ3aQcpkbPd1PlH
z9*5+i62BIZR|x`zNZ+#Paz!bQh&sTXAjGBcxo5tfEx;r~1}2cGBq@A!mIr2iX{~hi
zckyo5NlLrZysRoimPWkx)uJee8VpRCnh<Sl1z>lPEkOsf+gB_KbFSxV3MwwRa#@($
zYq{r04fT~(fZd5I_msM{N{~Oz<s9cC6p0D*D#OtT4oH$`Ld^r14H3spd)b5<eOa7K
zJ0Dsh46e~5#nSq+8B5xX#*l7%SflqSTLe#{_rxAyRkC{VounsLPW&2a6P4pA?0>s}
zOf6y^Lh1fkV0AENwj9ejGak7{QAdT4zVXPKq<z{+M$wK60`Xp)%nxqYaK9Rcg^M)I
z3D0cUOYA5S`eo&HLprO(Z;rPGxTSr1-E-r$NYjYc>R63-o}i?M3Svn+b*2#4&|MuQ
z_Q$(93-*07&tXmaFxN4_g*7vD+Se(O*^ZlncC<E|uII0?Ew^?L#6C)*#F<wcj!EnD
zw@Ht_K->m%|4X1hN=Q_6`&VyJ<jSsr7z=zL(7F1P%imEO7P%_z2&Fh6H{&LOdDTi*
z5=ku-e=1?b;dV|S=}znaQFSKpSru3RpC=&+D4WZ;vPdhfw4$P-Pyt23iWMssRII4k
zAR!xM0YXcC0tqAvDgr7ht%$4=7Fh&j5i70Q#A+*V!OHtW6%{L0th8cF`~Uvt+&nk`
zd_I%=+~1w`%$b=pXJ*a>*fP)Z4NF?Z{uTMOWVr7gIpkFVM(84JCWn)5zmB5|wWT#7
zNb}nmMmQ};Z=ppE|GPL!PAlf*_|q><MJdCJVJYNNVWb=EFsd^539!7OUuGgzWRT?z
z(STMoxKGLt$~Wc<h5lJV_+bT8M`1X<7XcEInBr&qd#}I;On@{A*Z9ZvYCOW0qcUH8
zCGU)GMKY1_s9sE$&)}0Q9+8rbCipLXv6>9j51aH(?F&fMs`-*}{$afmRWO11x2t&3
zK*v77e4lkSZ^qVPZHpaN;gH)eg&RvUkZ1{Ktq`L>t1oO?Vxs=n7?Zx1{<+*=+b<Q8
zZGX(W{Av9`jDjq*F(Kwqv>>BkcIi}3AhrELDin*%>bGBv5QJ7l8gcw%*Wibo7ir?+
zm+H;vRjM<&zlpwTAR6n72BZ?;v=S}qBLUC=Yx$*q*g#&abV%PiI<-)Um5vFQuA0g7
zzqk&}Xv4c*SeVUFQ6t=V8`%m`-G;vDf-MUAZXzL*Glf~vWg>?Onx7YY<0>)M>X4}a
zB+U9zZ2{1Q4vdfgjRSKbEqah_|AWD)G-$VIyiszFxE{<nKvw3okt5Ilc?cl1%ALw^
z=`)A&X65CO_?7+p`tC7OhCMl$`j;^Rv1NGb{2y8zmtV{d4?pK7-o((D#==@vu5UjK
zAWByTSlH;D@ACy35zup$1lw%2!YuU)t|B0JOK(mg<8^=ubG)~J7{^YkD)J?_@=E1G
z9X9ip`VK!%1wd88sN(Mtf)BQO<mFW7`wMU5-6$RG^T3@QZwInvB(yhX996qm;fCAw
z$p{{lT4qZupcV(4R)Z`Xpq6o&X<no;Xyvn7Hwz+Cw2G3J8O}@5TaHvhR(90)<JoK&
z5F;D=k1E(24Dm<Fn~@E1ihTtTDMwgDU(6wD&(1hsBGB>^3;?hdOWUVk6bv<(FqE>8
z;a`ozvYAquMuH#t6FjwaTT>Ac+(-Z`uS`p5FU$FYZ2Suoz@QCR#rD<*%MJQl1P}F%
zBL$Kyxj-(=^?OFyyEG9h!Y}UN-AXUp+|=_yAr=jTD93&k&3^={wpk~XTKLL4Q^Dw!
zHEZ2G8U%KxR=7gp{JVK0-(g1(96}?w@~1#np_&_Tr<iuR+#u9iFsn8}FbdTlaLYvz
zri-QFDKyUp!jY9zrAjo=NW@YB`7LS<aY#6vTOFcwtomphYCc>AF3h51ajHBgtpD5^
zgPJkrvIDrn=L%Hur&gC@ip}r8Clyk$A<{88|4SeZB&>av`HV5VTE6H7#!9<YCCI8*
zJDLmqj2K`9#s1Wt_oV_&CPx|*9~WeS=vJ{uy*CHIiXt;1Sn}cW+*~lIU&h*BssB=M
zR4RR6C1&ub=6?vWFyXSVNo<<@K^~YDU)xdgwcgYkL~+0tFZCnAXVg8IEIwRe4WaRC
zwsWfxV|&;kW!vLtaU#sM=n<tP-zm^YL#3B%^V>&)td!O4kelSc5MU`VaG`4OY55>g
z8R&i4Mp5Yp2#cyvT6kuPSf-9W!okRl=$)P$i^r%t(P>FL$vL1Pl?bB&joJ?rViD>d
zpQ5mMpH~RT$iz-0w%qB(VnP{%C={kEe-cXmTP%zEM~VO}|FZLmz}2$^#7bR+ail8N
zar+9gB5^(s(S@oCnB^d}o=MrXx2s~0O0={L_U#{IjK)EG?5`1KnM2f<crNQV%Y_<o
zF$sn}$E`6yLqa)oMJYaD@UUfvPVwgxQt^~?gbx9T9@D>E4r*n}W$O(wpHY!Q=)g6p
z=_16^LKs7SSzHNV<tZ*J!iWvMf$uvB(6~k0D>%3w*z_vF*0*s^P}Lco5C|X4Kefai
zA~G3S?T5?1(NCR_Rz3WP7(?>}XS2y^q%g}kLKW&-KS~IVmCj1)syp>+xn?sz#C;|P
z*w&1`;Wq-T_AvZesq2QGnhX>%pH8mol%n4vkcbiNDzm5hkyBEBka5Z?tLd8kHv;h^
zgNiWr&DFd{td4lhF5DhJ1EE|g0|xNLML!3#EEa3Ug`Hjia3zzEQlB}^UO2>E=s(ko
zaRJL-OwmIhe$4}*My!%tl2Nk@Sk!8n3S(t6j0yoJE$z$-TI~S}Yo#Y3J02_iWI<MX
z?tVv)rg5S!_BV+dm4)VS1X;dOs-P{qG6`T^J`?S@jz<fp4@1GhMXm4X&2q~@1Dyb2
zyIc(zxrpwvm6)(Bf*?1&wd^?f(RKJDL20FQ(8v$02e3kN3}4k7HbLDe*r-%kO6P}(
z0XV-E3Ok``3KQmlMyDRl=32@~+s%r`GOCrsK!)Klh9pdF!Exh+MI=;L%l491kYrE1
z`9I573su9GAcoFFLkf(n!yzl2@nEZTHP=N1_@OXM!%l-%`oj+y6y3DQHV6N)pr|$E
z=+K4#n-H2mhbLKR#-jRc-l+;M?&Z$oCcSC&({>m7$a!<@EBh5ERQNX^=FMV56yWfH
z|42y0sT^v|_R}8$fK$gpec$O(ULxky7E=USx{W*Lf?6HfO0BwbcJByosUvM+fo_p!
zBRMw#a*Z==7JM34;fD?22s4`f->DdkBy8;smVh3LA+|7-GTiuQ1X-4`!3KB#N`RHB
z3QJtj`=!_m8%iP44gpccGlvW##hw`B=<S4HH0zSbVt?A2qLtaZgjuc_W;jDz?VlH9
zDROt7cHk|TXP#OXHlfEkkEc8_on{*5N5lx-g4T2IcoILBFD-!5vySuMUkFAGu(hh#
zcYTT%<1xCz=+c)Wff4<XA+#+nXitiTr|8EuT`YaaUbPOW{U@OgYlU0FT1=z+$4qvN
zY42*d*=m_EE9q2Jl@#l5LM#!sRu-yr{Z9<BnXrH{&p-HdDqLwHoAk2%dLhOh=(coj
zTKg(8o&mK{1PMG`5@PeePWoVd^Ca$ey;ZMpIWq~Q{=4;Nx#u86kg9#<FTpIQVZSj}
zp`HbSkU6p!|J35tXOwF)r#<pHkZ4H2OammKbahJrqfQnXw<?;A&J=8gMy3qrK>}oX
zK(IgyO?Orwho%9BlL?x7J}B6-R#1jwpu*1<V#F|me4#6!Bh+#fZOx*em>UPKDB$E{
zk$+5xHGsm2EZ+HF3NU`4g`skgDId80L>zP~({?hL@jO1Q9PD5XL(Tny;$~Z38uVdn
zgj$xdLZc%iRZCMyF2~GJDs5g4V)<a#85Dha1%M<IEw4e$6)Vj{*!UsV?z$=!2Rj3$
z2VE|NYS#3CioM|lUL&FwmuM>-jcs)b%i%DhL>I%XnpB%Ku8`~C)yl#iXL_t4_yjcK
zSTqjY>)bj{_LQ*20p(@zX4J<_%Otk(uu$77tQNd<amg~}$BJDwH3!xTfW<$ys;H{0
zDBG`y^DNaUTLbd_YGF`^>5;Da@~^~xI9LFQQGLBF%%XyVXlk&z*a&1bfyHU81o$E$
zWB|Q^nXS5bO)8Xn*iw$1<q0A&w8P`IGv_ZGdAGvR-kB)LZo?a!0F7Ky+%SolO3+p)
z6<J*@g<*t7JjT`T>Kf@zO3k-r54#*w8B*R<td<lt7;Qg)OQ02uV?kw_zPu;M>QIdZ
z0XoQ)AI30iKFI&3pgkg(4nW(COZ?|?bX;~4ltZgG<AD+*U9iDyoPSgAmN8~V(n<~7
z0%CbEc_Z7WZA}GYj$P#s=-s@~({RR^rL1j0R^1b_CsDrHFQ))ZD?_jjJ1i0vK`Uu@
z=pEJY<*Yx>h`wQ&4mBgIG&}ibg_eS`6uCvp;=c>FZV>j)fjy>LHyA#ce|8G#csY8;
zO7kCe-#TL@vt<8N9G1~auAM6KX}bW7BK@?bsGZ)8B0228VP`^5Ay%o>3L{V+Q+HVP
zhp|u0eQ68xhzLnpk??DU7^wgQsV2MvWVvEK9*(H{3A@3_RblB^NB}**`)Vqj7QwYz
z*h^5vhs;&z(sks#f9#QE!yv1@Cx{-sq6mfjy?ZQXDtuhN8qX^NEcv1`bfcR6f9v&B
z)KxiH1VV0v+yOP8RoMDQmyy>RW8sy~1MEVDCO!5SbA&)I#@AT1Vg^X<`N47gNVIff
zp~o2#dqpG9T_@UWbCW;`YQ;qi?0fZQ<de-ys@7lo29VVT3ZUY!Ta#5Be}iz#Z5bjM
z173;0BlfE!kH{(7d{XKszL|=a74*<4)(EoT+4OWY-D<y37zN{mYO$~V9WN1In01`l
zLdfdn7heV6%x_A4$DlPk(fkKOEK0fv?W`^HXEcLZ?m3`Ci4gfE=N9}PJgVH5X3l9j
zp~kFZt7J`J_S-3M$WTEoK|*)D18NBs$Qz7im_-Ov$rirYD)_X9!p)0}b!tS}21jJ3
z*OMalPQ_{Axe6I2WUwW0SBwUOH1wnFFs>46!F7*?w99od#PSSR9hdpX1Q}<j7d1n@
zUkQ}NqDwS=k3Xa$LabE7_l1It?BfcVj-r?SW6D!`l}tTy*N`B_N0i<9l=7eN+N;cN
zF!9RU%d1h5BS~zVDi&Zl(IxW?AbC~((f0tY5EGc@u{F^kt1QMpY>Y_B244iPMjIx-
zy0$^9A_28p;6}l0?o=sk?NdwhVP6wQN|jVg8`0{OM*XJ+T0t@YPfzHV2(bvmL`6Es
zWx^~=Y%Pk-JiZ{@axc52itsNnREeQX9Z~wPAdPownGmAQpHrc-brO{E=71o)Ol7@D
z*V3H-KJS*-aHlUlMv<R?0MK&AqyRM)M4$J+VALdLB4z01|9GQDP!Y&h&$_?xY82DX
z_i&RVhC4q1wf@}fjkEPOOk~3ciysXPEtzWHE6;|>s$7nWc*=)hMl)SulRw2jEX3$g
z!mYk+=k`klS=x;AEz&^IVNfd<(a|%AYa0YcJivI#X&is~M?l6rC@WF=-w*?nP^NRc
z*2nm;OvnT_i%*MUgiAZHZQ$P*LM&k?za;4Ie3D9u#;FY>>`?;>l@kPfKlq5flMzN6
zmRF*8Sdng?J*c1ZodI+L7fa00@0N_CDyJhI|5qR*E(^Hoe<Wnk$j&%w@v@P#nypga
z>)pZ@U>Zwm=>S=gB=w;TJy)KLT=^Ow!c3~WK&uiuS9T6jC1YE~hOvK{4c`(&IuTuo
zK6}RJmIsL;RV+3C_Y2-F!C(oi+TU?3g%n8Ssqps;u>wtE=P)%Ejs9i&0C8#kTJORT
zUjz`l?tQBAo4+(48eGHfj;sC-KtHI7sF}yonynx64`3sOMn;M6WrCuD!-l44sTOD~
zrF*o)dPs#|5qm{JN@u~8(Z3?l8Z5JCyK7Iq#U9}%c3ds7Bm1AcThEM+lW{@Tx$^u2
zUjrLqFmk~WR(~=^&~9nke*6S}j74<}u*^4plM19`o%H5!35gmMd&`g>O8o=>0;7qd
z3Xs*+ReH6uvt5XNB@_HDCxI+a#bs4_j5k5~y86GsMv{tr?%B{53?D!X&Y4Rj5sn|o
zpHYeaD_F9oL~Dea$D;D;3Cc-;tN^eKD>gf0KUy?b>-bF!F<!HIx5EEPP*gP>gpgvi
z((jM`p?0g(D)O)09WC@>4H=E*zajwJF&L01Pgf-H0Vz1oRa(Z5I=xzc(Egy3cqk2k
zzO8TAJpN!hZ$?apDD=NHqSG_LQk=sMOsq)?iiUa}58?PlST;(uOL;|oSSZUL0kV>?
zek~yminaoo{jx=jZE^;}5A11T8=;szP0>aDqWVf1O-PTEkcdW{{+!rbxPeSPLBJ?B
z?WB~h?lyU88SXg!iOH_8Q{uP6jR(ak6;u5iQT!D_SfKhV{a!&<*^KUGL05(O&{IH-
zEF5g0Bg^$8W5}9gzW1l%i+rGp!SbNLO|KeBw4{d(Jo~7V+i>#84!l}AG+t!JaOK*=
zok5Y*`droTO8>=aDK9x?*!4$WPIdhBbkK-frXYecIQ)Mgxkg`%5hC*$b!)-o0ZP;m
z+6q!Y|EkX7{y4(GH)z5!|02+ubv9?de3{-YE_SIR#v{S57G`nj=#Tb3{W%IIAy=&J
z&ah~h@MB1%(zn-plzdPdU_2-C!qIQKg+jkc?=Y>M31q-L=S%=AuvAHC=Hc|gX8}TL
zNWx^&->+ArCL)0}LK2VLpKVd8cV-Qk!%yd=yfNM*2?o{Rw(~(-Mi`_Amgd6!8~$Z|
zh-#e@Pt70ix5tnUf@m2qKoM!!po!!bVU}llUq(~9690<Ann#XvNI8o6|2RH2Cvh)&
zzW=u%BmLy;f?~g-OPmYIpp<^CAd6lKRpyl(^FJie!eJ|tn@jX=tzSB949j%|F#cyg
zsP&EeVniHEE1V?+2FynN>?d5n3k=NN1km{DZ}SGprGcZLZGR!J)~T|aLLy~2v(mjM
zuoaSxLMSoWW#0-5=bY8f0v7cu6hR8LAToVB4xhtTzHsL9dohNw4K3BcpslcKO~VlX
z=L)kD(<L%IpZXmDV~dJv47AcE!IXW3Qmq;^3)#JZ4D!r(Qyvw2FeJ#e_d+R4e(5Ca
zMVhKC?IH`Tj!OpP7|HGz1YLtMw=fG@Bg8Vot^zu;$9koJ9IQBV$;9^oAZA%{wK|i*
z7xQXV!@L&~2;BSNC14TXb7Z<0?R%bq=CNX`HmD%#29H)sjZZci^K%qDLJP||X&<H`
zYlTw3SIbu2f=lBRn1gEJq}tEC4A8iQi}%FfAN5JaG!{GJ3`g3(^xex-{@53DMgJ>(
zz7R63HXuD)Sf4p55Mr}8CrlgYuKQdGMA_NC!8HOtTd$Vjg?+-!PW63Lh)lmB1YUd<
z2%N}O=A3&jxtcdP9b1-kfO&qo-i?=}2G1^uZvFX2xHW9uw?>X6F>qgI_+Y$?U|-2)
zHri)zun|}5dvwdituc6vbEp*Hq~ayV=+>A3>Ks#gKai+j<E|WWS)m|{3}!U@6A#1~
zVKJ=4_LaBdpv+xlBh&Z)fyGb1#daMI*!p{gSp4)Q;T+?St^tB1h!q?Qerf=3a2yHg
zCel2gIVcqee5tDfe=-=vGE@TFaEU;HFB29KLP{bux!U^?;Fj(S9c);<Zw)eDqQ8ep
z185nWoSj!)kv-W@h|`fi56xzUUnk7GX=zo{ncqd;&}NiYFiY0H<j;j!7gju;)AjFO
z&%3e41Wn!x{g*<boE4OzGu0}3pCJ~EPOXrYb>H?z-eEAN0J_2Vf*<l~iRYlK(D3os
zn^RD}Ru~xu17zfZr^e-@eKpL;Bkrl-(6Lsof>+B#WQLb0KU0(zp0R}+e>p}KuxMD)
zfvdoQKJg=bT5y)}B5PXb2((OcuUC<_wswml-WA;6D##+F_d;Sr^PlIR78*5KwxhDF
zKIE2EY|IM7WgX86vOcMZ**{wA+W;VT{uEh0%lDPHn+J|cgniKdR{~m!Sz0zF7~v}#
z2}oL6=`~v!#X|6=6@Bg-vtA^CMh(sRb?2uK7#us1X-}p9nSxoPWhaZyrdY@)q69Q(
zmxN8SPYW^9a62Rlj9h=pC?JcT^AMq7`X0i}+t}>tpys(ukP)U7Ll_J@bA%WHQC72y
zma!f!<(;5Ln{gQVsrLN@SU^?*SuFA!g;+q{mxw%}u5o7+Q2V`@DwX?x+=Vye7|I<q
zEfT8>?gl2mR2maZc9>9W+sA-G{|eeO1~L4Xdbgkq>fr>DnofWt#$Y*B>q!B!ib0Dn
z4Jw0dE09i6*2ziZE5&Jnv0lXO&>W`})It*k1uPuKNZEvJS#3M>Ui=t&nO8D%A*!Fo
zSUA`qecN02rQ*T_3@y~3dOrwFPithXFQ;XvJTREBBJ5wwPa$-HFtzVE7R2%xtS+Nn
z{KYsRD`2TCL4{TqK+Cvn<Dn3nE^mfZ6c%ZvpiYQoT=pxpE!RI3M}~wIHMb{YkF2xh
zvpFlzKQGL<07W1DNC2$PG4-gE4J<$SBl2dca&lTn>ADtxP-s{-r(}x%wqEg|!yQ4K
z^p!#^=HlvH77>;RFy_yf9V5oL3f=d%Eea~;w|fct72?yVquolfO?15=OK$39tU!(D
z)riHttGWaZgKM290$SG5aj*j>n_Y^AnpfBm8B~+D;5DLa^*CnLt-WDJw4AkJ+DNMy
zbw&mOUtMo2lyL}yK8TeS{xqd-nWozySNZ+|K~^(nMu@@Vq1aoFELtIvJ|QToQY6wU
zmK9@9=ubGR$T{&iE(!1&Vz@6i6+cW>BBLJul5#L3g4ReY%*J1?P;#QpqG2uz4PK2f
z=&r-LC{2oe_X_+`X;zK!$07k%Dwvu1=_LPH6%bWXiGgug<9m~MrH3oWVqcEmHkmhM
zt-?IYzrLE6mhz)9m(;mpY6{CM(^m1a9K$Tn?9yS`JnXD*<ujX`4w(g2_*DvR*?}*Z
zzL)y7g4Crz5_LNpV=rtc57W@M;xswI$vlaCd?3uyRCfX-d%NF6k&zgeu;u$HKf{+X
zaXw9^)E^XJb(LGpYD>N!^K%f&pNIAht^c&tcW!r;hUe1<KrGA1)H<i*cPCR2f*UIV
z`MyVu#lj?s#or6{W`vU*9xO}@ihM9o(eoAQjqIZoqobhX3kxAH%`j9|is?1FM2T)g
zYg4gJt}363GGbh7ERPNl)?xWZ{;Z``RH6Mt6ZHRq*_<#d;%?IlKTJMbxT0K!!VI!3
z+x|azeMFpael?PW&g<MB2Svt$rAmE`AZk#vdUo>p&-4yOIiZW-d!R0r7CA@OE%Jv1
zSz3shl$^;$hnXpFGFiub!k5efB3_LwGzGt^-rm*Gun*T?(ZGAu_vzLIzj_XU5dewO
zZf5YCVh}+SaYn-85rLKi77%m;OQm5Zc{AiiH#AFC6Ff&8FoC@z^x3VTaj5*YBP&2r
z;<6H2sC_4`(y5|4wJX6e<6K?ZEJ=WN|2O5xQr93Hl$Aj*ddkCmh=y1en6zn0Gp+B&
zu(T)AT59Lh6@o3VVK_$4e_D{0wW>6_dhdcqVs8l08e<O&GHPm*W~HC;r~$eS96JQL
z{&)mHe=T0+`Jd0TSTuuUcZIAFY!wu>ud4E~?3I1&3G+lKO+B=uc%C4m0Ed)yIg<=n
z1l005sgP~9SY`50$fsen^1?hBFj^?g#vv9SGy#{9dB&&MFEb&@I(h!|CsT=6aK=f5
z9?}YuNlm1fje;x{1SeSl9p`t&-k8M(+NLc`MTYrAscZbRF~m8PFtOV$$oPP+pSCLA
z`bwaYB`n->H}X>;MuRYgvE!uYJ`GCW%&}ya9{jn_@M=uK08!4>9@_CqHUCK;@E2CA
zu*v-^UabHKC~!GLLH)B}#tGTl*^O@X#Rw|NMu?G#2v&y|<IA$aCVE*%*5YeGtql2E
z=~NGEpp`)afef%HuqDCbAo`ptgQA>p%N3>)O8xcnW*EbOZI8KMm_=4D@vYcTj3c8{
z;p(DN|MM8)6bPqeCi`iEj7!mrRB;Kg06Lo@cx$YGMsE?F^D*y|>(|C{$O|eYze$jl
zPU62QScX3b2t{=>488t(y;{*Y(X7k4{7ymf#H!`U<}*uxEk~rsHsZIQ2Ve+g<+{?}
zxs(^n5QjvN?fiBj=D!FDr^uhS41f|$Wppm^mn=_x9*@>F-*1ZmA~&8ZUzv&$73E}1
zPt*vp7C{lQI6zMWyZGa)z^xEeE*dZFtHux}@wL=okg<T4={e;p{%_@xExIhy75c9g
zHkv0?vG+*}GF0!s1Y7koYqAMZ?-#(L>Xq&&-{%Oiyi`mUi{|+%K~@h`LUe}zF@Z*&
z3UrX;{fN~77QMC~ha_)b17^`{3nX-Yd}GQN`UrXj`l5Bh@I`~+bcjZ(9YQQ}b{*rz
zZvwEgvuy#HvzW=bzaU0Y9Ld_57Fn}5rvh`Rj4}T)0hSAH4MuI?-;0rOS|{88XFGl@
z_hX?et*gS9?@VD@n$UH1zB0z>w^6-v7G;L8mYzhKFfBsY3A98RIP<WRF~3z%7Vbm;
z@J1X|Tk>pCy6-N6O46zd8(RI_dN*q1R7tJ#Cjl0>cCymuuY3i>vagG~;m|xkURcBi
zk@7`a;j?}XY)lw7;nQJ3_Huvq>)=)jtc@~GNq-+eqkGuqY&)p>$tyg9SY6GGK*L;r
zL1Z8--ZH|7H3MeZ%;9ht-C;mlqSKLvz$F6}*2*`Y@>TmmQDh87>?UKc8mGQ*jIsP&
zgfVG0$HxJ=@P<}XEx=M_8Cw@!d=-0{fEbMdu{SL5wj(S^bDiHbh~#whRwd`T8=F%Q
zJ3I0?kKjkWm5Mv~l0702BYJiY#^t6m(6spjn3a%42CZ?ix@f3X1$tek->jQ6+62#3
z^mwcjYiQ3<;Hd5Cgf>!dfUHDZT`RRfJ9)CWf>Jm-D{*NI(>rjs5-x@b$^{#3%bCE@
ziwLj?nPkYO##MhzA?jn<bNJA^AmmG;fb`+F>}}<vT-jPC#G*y0oxoLP`^<|DoRw8@
z8q9b7b1G0UyjSes6=EEgQ=K=iOiE_|KQS0Bn~>`O8fEFeR33&>c_c~QA1*zg-zOmQ
zq~i{f<A%KtW)!50r1vWK9||&h;X_8gLZ#ny0N9#Am>sq(l&=zQv0~LsJAaD(E&m5b
z&)+XBZGNRUO9SDReJO>0?x7SUO*EEg{m8>Wk(JNXjY%QpwfbUYz>Ec3z0_6$XcB}h
zaUg0$y>|Fu3<YNfwT4DZ#bKxnDs#ClZON}|6*D?L=GRQE0%#RpSsm=NwniWlT~p;p
zL}z~30502{e-*+o%k*3uCBD#WL>1KZY@R~C`#2Sd#+C<Hye=f--*8=Bwm<So$`iGy
zsovVZT2$;iN)1iT!lFWIJDf(mN`K*JsjzITtHMg`{G%y<>^kLyAon?=cl$i$kL`RD
z#^?Kog&2uV=0z~_j|jBfu?sMqfL++yA1cqVNb#aT;|v0`O-Y>%J^EY1jRwqG!ge8_
z{Y5Gvbcc~ig%<=yO-%D*RMbL@2N7JeTkiK8^Chqq8<nTdk87O%;om_ay?VcjDgM`A
z@rsyIg~rME`3EoLje)b04a~aDBkXYe!#@qT78@k>fR~>Dw}jDap`fC(W)usskz$;>
z1v<)#*e4sIf|6F-xB}e5e()IjTWThtMqjF4Yf5M#suY8Hovih~JSO|&`~yNGuVK?M
zarixPVoZK9*Fo=%klX(o{2JldeatxOe=Ed7kJA)95a@RTE%#bxW3Q<n^RE;Xjszew
z+#5r>Xir-hHvb!6)@!l73OdxC1c21s`J+3AD*nr>rC(A-=TYgGxI0^>%(=3d@`2v*
z!J%bFP2H$9Az=uVFJVg@3dZ+^kiBrqB2<$9M39k2Q%R1gXiLYKbWmd`Z4wA}{#s!<
zvWvwHnLa!Amm{+k>eusy84VE2*-cjN%LEy%H141?WwJI=U<*l^AF$^?jt%v*36YzG
z8JnPz)wSs;<V*dZ;uv$VC@(!nzW+p^rHu9_?AGnm2B?LPC(&lY*55%wEkDdUILf8n
zroRv<*@oVLd6=)r;+?J)!G_h8WqOO^*5#$-;q<mZQQWF*W}9q6pDHveR%Nwjcx~H(
zSZc_ExtQ%?mE-U!fTYIh*Kx8c(x*eplUCCUkn>sh5NPe488mW5Fr=7oxK)o7n)LL!
z{>O0yvIQ-b0A~*c8uiA7ZCIuYygl|?iG>pxzRB~W1lrJ=Q#^^yrh2zTSQbYngr)Lg
zOqOxz^Nan@VvNazW?iNJzfoNHVKXDU?m7~e5oJ7^|LAL95)yG3b;qHneE*%(EeD}B
z(U#pAywkswbIFYFsTYev^Ivoa?+7uvhPjD0ru%Eo0<=n@Wy#{U^8Kbz#xVBB!5I=R
zHR~tyH=dn}LzA0Q&R$fs28olaX=UIl__XvnlYspQf2+b;5$LkjBDi!wnAKRc8e<e+
z7N=wOJGB&YLZCJJsjS!`s+<D=lZ7jxt;@eokR`${WDc358+=8WWd-Y9B;Xs)i+z=`
z=TBw+pc|OQPIHArmUjn$oc**2zE3X}FZ%;DJ1g}!^hjY$uCx|F%?XQIGvWz17x*DD
z#ALjnk`>DUMODE`X!eCsyjCD=ke45{jsY|RA!Q>$)0#D-X%(rAjLA6bP^Dv$(zwRS
zPG!EoQJ|HbD|E{cEpCakqn$LYvbKh_4F#*G{#N<2bW_Xp>V<9>ZdBp|W)?4h6#*6b
z=oF-nTJ<e_7zL#NmbJPW-v)}9R|jo#xvFfXu&5&qm&S(~U+#se=!;o}VQ<$(DQ~c=
ztPW@>`r-Fe5i;CrKFY3WVHP1wOTQ8JN)HoiMA77yJ#tjs%>pR|M?eshA>EH+Upjbb
zgVQYntsoPMtF@ZpM+!1(qF2;Z@a{OIv^mwpm+S9~L(*4p?@6AYBFK6R6t)xm8+vEF
z8qb(Vjy~3F)HibFlBQD>GxK6l%K>w{a9-s)LCU%jgPUG20f-ngzqC9k*o!W+aL_?E
z!F6EqhRdUHMf4mcC^@nPP!w#jF!fbxIah#?B~4b@FH(D@c^C&rU=9i`^N7A6mPw8z
zqwZi`1eLW=%Rp`w8%UVa8`dHNOY{mt4rB<PtvZmW&^c;)+X__K5lk2@3KXp|Ak)V3
zO6FQ*k_{8Wt&2wIQqDe;a7eSqRlr87(!%0!Y0T4ttg^HXnu*7kF`}KaOqGkSP6Z>I
z7|i2rU4I}boX4)2u#B)H0;Hb~=My_$YY`)3ilGWhe72CN>9GpH5*J&aas(P-5N=p-
zM++V>%cj!Pq%A)VtCM21aKZ-A{7tTAc*CGif&5WDoTik@|H(LZgej>iSp*XnHGwMG
zb`0zO>*543a+6QbMJsJ{9bSz%=q{N(`rbn9h%__WaB^K2`D)g5l|EUc#~3yO;)cPg
z(6EoTE0<mm!sxGEv*mu@4ZM+ksi<`lmAoAu0@Okclo(W=pE3;0$Tpsyg2^T0YAa9!
z7sm}~paHb(V&`5b?y40wrlfWb%n)Q;hxI=h{0+iWwSPz-EdLc{)m2md!*SqnPLnJ@
zEy(i1AT|LmCXuDFzwvCSL6PELh#{6|u-BRAUyUJVk!<_S_U{RzvS1M%2zc;jUZc#?
zBVyjr7v7SBur9&=Q%=wbvO;p(h9<!?f1E<JwnAF>j{q?;Y7whKzV-{Fcv45O29xJ6
zzui3Lmj-D(^sM`YMS>yz7}mqqgatoI!8CLUvkW`cz7<?VSBiw8Jvs&(gRwJKt51?c
zf@jMU<6t(+Gf@N)D<UpYv1M4R4L%KJR++=9xTKv{Xt1EF^AfrKaplY^1ObojQ=0ib
zDbSKaaLNlO#5V}D+AS{|&m1iP#s_4Pz6Vj0txeo4IWpz50`sT^ewhGzWgVSS^e^b$
z(qtP1m-%SO+bqzc!&IXd&!XAOxKVtFn34!`3;jiRfLJ|OV5p;#!}qK1OhqYu0z(x-
z@81Q&9_0nqbiO|OE(^vvv?u}p!A}jMHaN<Hc+dL9J*k*5bpuYx{MIq%Pt!$=zG7p@
zAgk%17Svf!m?HQzII5+fw+ygViLGgc<2eB}LR4b>hT92gA^|kw%R(NTYPrZcPM`d+
zKOSq#F%&j@BE2@=o605K1INWm{I>g2r6F8&<K=z#r~I+oo33zz&y69}3_&K^o{cY~
zJ4V(?!}rTeg`>;TsW;yyABd{eLMEme{IAAZAoZSXhe0>_!8i~r4%f*saKjJ*vvSKq
z3Y(-TPymfY>`!EOSD3rD@@l5}MUIaYr!j!0NitR5r2r_B;Ht{iKxwl~(|<hPGDOWG
zB(SB<FA`#jGhoQ-6cTbx2_PLS(_Ff^M<?(GueaP;=F`dnT6&h^Y7BJvGh(Pn`$cHC
zXT=a_8>xc4){w-sN(P-C@?`|k$;=>CT^eH?;-_ze+IPmWA@ef7FYzx4vNp)lEHwM3
zRXE0f-N)xouSn%JoLQt_FB23sIKpX5QicDyK<ZbP!ZB`4vAb1*Sk9P3*}2c|!YoN9
z{Y6-N6+ojJE9FMH-!nP&9aSi6an(}+qO^2Ni~28r^UndTcBHG*iY2W(!Y##;T)SA=
z{2Jd?R<X^t%n+k~F>P1J*nXr4qkvqq%C4m1)|fVv&`0L^TjejRY*}oR`8hvYpv8x>
zn#}`wzCsZ7-{-PES6%8W^=4VX?2@)+O^Xq9|H{+@LM#VNFwv4ij1|HxHH<=OT!aJv
z8H3sjZ*2XqI5gWeY(wgIrV-jm%^Dn1?-v51`p|&`?eY9dSVRjg-y*~PLl8Y@tDVu(
z9aQ?tWXdlpTjZ)-Kc^-I$%HIxXTJAzAnUQ%GJqDuOrFh}32Zb#fnAJ(Iv`fMl3W&i
zuu)WLpyg3zWAV7mKdhjJq1{DukNow7Fv~aNFelh@{X#)-5_!{k;f;E=_OGKp43Wqs
z{-s&wNfQtT<~8-a(%+bstZF~|LEfYK#fK>b*9f%qkqn>&iR}_bVkl*>tL^{x5U*A`
z`gjc0`5(?n1=2>)Ad~*?Q83Fi3|n4Cs|=9kQunAASB33x0ksq{^~{N`oP6^c1*XJ0
z=-j0E%$o`aAvog_W8v-|HJA6IbeSU2&xU^R{n%RtAZsNYs}yF1=fD~Ls4P<aJqk;Q
zr}O!x*w4ClZd{BqtTVtGcf?Q*r<Jkr?XzPD$w}Kx`~*RkU#+~d7iF@aBg_bZ%BZry
zl;*z|8qpJ7CX(|lkAcvlCNfTx`0;u(V(5f0SGD_h9#2KYWCZDnlIHuLKADOL*?`Nb
z{9lDoa|ljVtm=NWkXOr7Zkg(dj_a(a!CF`tIj<y$4+dLSOM*S%P*=foO9ZUFXk+==
ziohrt-MT|<1u&{WS?r)t=gXfJb8J;55|r;Eq@`e!LIPt1S+dj#3w#iy(x1Nw6nf|~
z$0`1`=Td%jqD-AFhf>BZO-05jDP87dkmVN5gf6(jAd<XUZ>nWhY1)GD-zs=G#Kcjd
zR#0W(c&w~PiL^8s4%#x<RQO6oX(0RcwPl*Q>7+P2NE{5GEEEW`>@Wa~(*_E^EXKmY
zJ-R~wLTe1cNZrD3g;_1-6tXg-4hh=-S#ZQUFcke>+Okw;U=}!Amy>@k(7b8a4gzh-
zau6c}Cknz;N(UtUrd8ndqrqT<W?`2$8W5()%JEOu09Xy7xi2h5=?oX`TAPYbt8#dR
z;W3jM@Tfjn@`&~|1=R9WP&Gw6Azx8=qoLx&R2_r}pmpkHMJ!kwVBt`)7DJca3fEG_
z(k0VAUl&F{*C)DgswvJ}5Z=|PYX1Je!YwxLqt?O1PlXsmPAWrMM&<Bj9GF5e8;)k!
z-(H7T6S+A+&9!e*?kV-n`VetG(~5B%Kl)gZC8Pt%=&n9opYp^J`p`+8y#dGyD~qnh
z{{77W>aNSgXOsM(t-Px(vyYixc$Z$S23dRL3N@8Vg2*;-qc{tD=#v;qOAR&O+D3%h
zom44aE8p53hhKI00X3?YvvsIKH>tG3(Ans2@-oAWD>=xovzwt2xAH1sk{w!D5>U2m
ziPC1iG^b*$lveml^u-cGI$`rHJ=tY(A;!w=x+Z=V{(7OtBsrRLXzQ3C8iOiqb-6_M
z`vn>?8G1@8eA^uWRs~YMF?Q4mF)wHYiYxt|T>w_2x#64<@?}sL&UlRvmTUoM45U*o
z_5rk9m*g@N5c{<9NN<B+#ni4fSfOQxpauy(tvYmvpfvkrip%npl*Ni<O?Su6q}mu`
zbAm34uQtqzlT#Hej@8A!;BOMg<YK@mVk9(T5g{%KQCEa9$UiC2;+i0hSEc`z5MyD?
z@rSlDc2CNa*o4)=RYHs^NLQNa!F(?Xv~q=|F*HpG?#!QmoAOMr8?CFH-2@1Ov)wZs
z?D(LWcNjdIH73fP`xdVeBctHQBAOWXbD@?3@u0kkRG|TnC<lLX1gO;f8hc_xli3a+
zRoBWZ>La+J6yxlQ)`D{fpU%?#e@V&Odr;YGni~8<uw2Y++4;&Z6uPAyXua0vvjJ}D
z)WXRSQa=`_#GFG`EUN#1j9x}P@xSFSss#FbZ6`Sy2Q9<Q3Nvx`8~q+{R(rAwfH1a5
zh%xq*Nri=lewP3%n~Xf8#;Ww63A3!J_h5e;S<BfEXcP}8Sy%z13(pg3!I3se1QG<y
zf^%vEYH3SyD-aDB!=My?V-?zpfxcf`_5Ju5(&h^3SIY!hDJt0dgc@>!kSK$+IBtic
zMyACfC$qf(MXqlaWTYoW&Xvd(&;)-%uw|OHYi-Lc_E*0L2ES=Kn?Jt9H|X86#;S+R
zR{69)0~zg@?qKxtQUUl8Ltrsao#WGjEU#3bt`ivee;`C0beD|%#nHR$aO>d4&Vzin
z%uFhaw$PM+02Y;xqg?FcgADHrjJhBly5TreB*DhMbvLPwCl#4*%RhIFaPBX=bp$NI
ztFtMu3M)4uW~<ss3>DH4F}(_?WejUsg>1yi_Xm`eVeH!HU~ztq|GzkSZ5)tJY@9zB
zd(6=#&Qj}rAk1hpwz`ZNoBuFII5Jz#g!;M<Q-1O|mx(l+EyR-40=qO_{!w8@AJp8Y
zHuTTO9!oK@6_$@y3A2)+htgg;)&I)FDU`~mw3^lVw?6_Udo)pwx@+cg|HmLmmRDS?
z%Oh6l%_xf95Dwi{8ARS%?s(Qdg>Qe7@}{HC49~vT5sO(f2Zm>A=ZBwySib01!@Z{v
zXw2W@pwNdKtPHY|5r5j|O)Fy@#rlOj!;RQT1%UPLjYFd?WH<0+gCdHT73m~PD^DCT
z=6YTZH!7TYD#_O^lYVv_Qw7s))#uU};@F5Lz5WA1Rxw<*P=IO+6X&0U!R!b!sN72Y
zYQ0<BZ0W?RiT@z-0WIj-r}*o?Nco@-)lrzvUmC*o-;u=e#NVS>i^pM8INJ~S%HD}%
zDnhlt@;L9-%ta#d%g|iyKl~clh|HV<z6>oipw^6Zxf3^Xpd2oQpy7itl{RUx9fck=
zfY#<GRI}B_0OJBiKjxHETc^Vqg1J?#QuxD4-8^QqL@Jf=$AWBlVu4+P9VsIP1UzDK
zCb=Y>?f)*n(eT9edM%gFrk?<|Oww^hjV(i<6*TZiesvC0stbi$tf<2zIdMQjm}QC~
zC!B5iP>@v->K17p{v*W7JdU;90^j$WR4ArOTE+F(3bK6Y_}OH}pc0>#epgFAQ1^vP
zD=1xsP$M|H2BvK-yJy0Os1axz6FWW0Vk;CaA{z}kGRw|iMPh}4O;|8uNcn{@%Y)|p
zToa65>H&d9x3DFLnUpLw-`57%3XbJl858j11X=Pj(H->-g+i^4V%;@$m&gD&GD(Tc
z$)tV}0+UiGb^cFe;>R*1?PnN-{E&8lE&V?O&52Warz90{1yhnRk7(P`FbzHk4xE$2
z(r1iSB8Y^A*?>j0Pk$zLW3_p-qDd#Hg@Sf**a<QwrMX(btN<LYlA^6s><dGcrC1Q*
z(qmypKBf)Q`=BQ&_B{k!9fnKKWgbn3;QUYY!E)4cn{uVk7ic-6vykBJ9~NZ94q_EO
zF6(f|Vvn$AIGuiaCp<>kREy2_pPmk4BQtGsod4!*-lG~SM`+fTcCOj<OV0t1Sfqej
zFJ+n(u&8H%N7&;KxM7*RMkGUPDh68_hsH*Sj{Tzc7iM{p8I0;OzeR`@S8UH!F3@aS
z?2WVfY8|h}k&R_Ggy6v`(%6@-v7!UL|J+n$VNMYyH$|;LdRenV(Dpoh8ubthIg&&l
zm+waljaZWX<#gpw2(fYngK-R%!_EgX3UH`H)6gI+-Sll>D+rCgblgOB@&Rn&Bo3Ow
z<gFlPPiRxBTSE+z7^<rF!{XR<ygr}4WwIYB(7KTdjKdW9#|2mgA+Va!jLR-ed6L<j
z3jcWw6rqu>@H2bjBPu1LC?evX?-)W~!a%G(<Mi)=SjG{GxQVaa&+iQsl^b&Dx+e{G
ze-|8eMvTpHv1Twt5k44+)$Np3gvn|^EtY5zFHFY+Z0XSp6L)p%@KvQW9XiUESgpvC
zg2G+t$CU-cCg)eP%1x2}Da=x5?2+AgGUd;@1k{p2j~h<!o)<%i!fXc0^+RI_nJ_4w
zY6Q__;AE~xz!0I{t+wf@F%ly)1pg9fWs@q6g^;i*xMv^WC@bW`Ox^q4SAa78;!sTy
zliBuH^J;C7NpCpt69}`kAGjqXWyUzB;e~#+K&!#=oYaFXgZrltb~ZCj^FxHRjLJx+
zqC%KTKd28z1LW|~Av_a@Mz<n4a_$dO@iE_~Z}Llp7?If&CMzbne(XT7Xe=*drPY7$
zT7ak#&`B|`WNvZq4S-gD%~6n@bt_3g%`4(H%SPi!wv|`Nhq5<xv;nmoGpR(JX7o{<
z5gV}ZSuBm`SYZ|;d;jvKqs{Y$F{mTyI@H1O34s<N16Qy>G&zot#ZBX(HL*7gc<bPZ
ze=UYs_mGV;f9;S|WWf@&YD19K53)2<13zSF$`ht?QP}1B^cz!<)_N4xUj;!=4n}Aa
zmFrL30$|0Y4Po?+#r1BtrZBASGk2zY3|Lek&VMsO^gZRxsKHDvJ8!IX>E97(8Ot8a
zDqNZG6UPR3Ndr>qhYPdV^4O%Jj%>6b8!{%9jl&8<988<pIg5w6&^XuxjxfR3ew`po
zuM*R2RMLtcry?4cOPe{McVi*WmeCph=(dyxnK|^udbi9VCP+u)ci*0ZxP&g4etAm}
z`IEGY)Z=IVgm)5Vwi`BHV*5x4KA0Bgv4h4xc^B_3tZExR%lzROWK>2Rl;W}#sO>%Y
z#ZpE9t!B7w8a_qM7_g|JqXyF+UrO8xtzydp#|71owF=MBRYm<5`;U~mQH%5R?6UG7
z$NAAHFfPn*|0>Xu)>#7VUu65wVhjmfc3UuKGCCC<s|>|l@zg<x(K46>kOo?iMJ6LX
zm;^5OZSPJ6ga+tb{2(C~P?KC$!eK$sGZH@hE%(`b;aKzz%sXe@&pY&`Mwlhoj&}cX
z1R|E9V$#Msvali7d;&7-Vf^Q`jo&EHXesHCH6*cNKrOfQ7@T`&&~Jq?^ONp2!Ynkh
zEd#lBk(p1ca-<4o8Vq%QlM=F`m6fv%ni_jaP{g%WS~xSrJh6QM2`=C7jy*AB)&=r!
z3bC0bi6TY86k#gt&&lJ9WgDy06IjXTq8nkx_~X^u{V)NRDyRCTrN3K9lq&OM?P|{z
zWCeq;lDgGRY!qtole=KW$oI+zW2!x&YHC5YpQbnSp{=CM<x722f%)KI1$I4eD&mdt
zh<=w%(!NBmRBu^%4y!xGzHhO8MmS-u!OzjFWmenyrxg0jN<kQJ)Wss5g@u$bJ$x|w
zK=q0jg7KGNWgvX8Jmrn&{&1Eyd@Ij1Ko)M9xrIj9$}=)6{UzKGFe3*0%7Q4y>?oi%
z3N!qo-azwEfVC@S!(eWSo`O_^#D`%l!NwC~HDa*crr6IB7}XbgkDv(+n0d7I8_Bqj
z%418Nm2)`AuMlF@%0^1;YyUcq>%z<XT5r{{3{O_E(u1k%|NewjxbYYp%J#hlP#=(O
zTmcsg>D7YB=rFbb{9A&oL^LM5qLR#?KM|OUpUgg&JpV$uy<L7W>ofj)lkM$_i?87A
zylUQ%IgpK5YvxQ~r2ZNwoZ_d<8#46|m=zy$iWLm7uxyJ*@)!($s8Rw30kW$Q=dq0I
z3!>Q6yo-yp2PWSa#uz5jDyukF=BEj>6p;2<*;n8Dpg=2jh0K~`W#pyUpY+`lCn|mB
z)Re!n91><^$r1z|=&#7?<$8<SG=~a^`W7LU9;21C2Xq|eu{X&O9KZaHkd_w5Dh%^M
zh}=I3%2sggZSzB?^J20a^YLKb2rg$`xfa~m2!%5J4V|5^)1OE?<>pJPD#|ZKL7ZKD
zDfj8IN9IzDf8<{Z!_z#GQc5o!g0fTRJu5Dq%;gy059Wh<V~!=e^wRMEJ_e3UDyLMQ
z-tLOaitNKVi3`?!nz+pUz>V7a?+32EE7|y$HWOZ{9XQRo>#j{EI@c%C9!(s);K@Yd
z*+l0RiNv}@VoM_NO2U1a$oT0ExBO&`o<A9S8_&$24j=rJAC37Yk+=xg3->KtZ`>u{
zB+~zqocB#4?bRjvIem3~TH?_(_exsg^|b%C{3YIQ^?P5M`%7BJ1%IiTb-|};83UNR
z4Ei!HeL%&aZ_?TfxU^!>^z@7Y$P9zd`{%BXr?f3Csl2arY*Fs%AMEPrHl{gO>-3jQ
z;F|uO=C))wH|tyc{*d1To%@-7e`feY`aQt8=Izcs-OeR{iCc`D(UrjGI#+`m*wrQb
z;)dgf;*x#9anIwgf#<-x(%pcY(%p5Zxa6RMbeE_j3yFkFpW|FwgL4@RoNF`JxvWLb
zWiE8C-Ad=$E_UwJM(0jh>RgAd&b8m+Tqj&dexJ6-xz4+tJEPgT)8BB;?{)6Xcbq%>
zfOBW<ckbN7&Yg3}x$}=Ycis`_x*m6~%Q5G=pLDL<3For=wR8F8GxxfVF6WE3F8`uV
zE-%^9joa7Bjs2pNE1b~Y6<pTd6@Ag(jo;VamE6?96?f_2$~vF!N{c(V8!5+x_Gh??
z6=%5e&(3gF7j<@(w{&)s-#*PvI<vE@e)}vp<*Uwa3~5aL>MU2qvjq3Rw1k`4-X#Yo
z)7<}_>XJ8{<J|SHI5+er=Z4(x+z%hkbT`%}+^~x>-A%;__oIGU?&dGD-0%rm?v^20
z?#ExWakn<Lam%6WZ5L;_pFDetyS;OU8|6-MBX2v!-T8J~cSq+_-RPUzy1TZtb+xBC
zclUs{E=l=o7CE<%G<%be8HH(X`p`62(>INZOLMbI(%sBa>8_?f^^%|F&L)o!Cfc}$
zz3Hz0Xu5l(opUP)+Y5X)zc1z4&?nu^<#*FQ=SC6dG~AB!T=EuZlguZs4#YXoxfxy4
zT~p^Y*G##l_riZ=x?56{>Gt3^xt=`Wn)zMBx24qSUVb-ylJ1)CPjj~u*9z+GE}r+|
z@^A;_w~%<=Z{wz)mT-r89}j+%=Sf_~`4pGu258juY#Y~%zoxbcx0&~)<Z&ip4!Q5U
zgFpYS`=I`&w7<0Pm$nFe*4%{K#Iq5X+@F9pr14ss`*cWN+5)f+#QS+cURup*S#J8V
zg!|&{hte8GLD$0x=lM44P=ZS4y#r;gc5V`Rs^OVjkZ?`?60Z5ogqyuP;eHOjoxCjZ
z3HR&yXS-LQIn(W_J=?u@+u827pPlV~lRVwcA-#J1{SH2Q^Wa&8J<Iil=D**5fqUzj
zbKN^l7rM9i^>lyy<U+T8M8aLex8(=2+z!J1&-M%5{@1_j_PzTp_vf#_>v}71N69ns
zHGSR2b>+P${${<A>DG~txrE!wvnTij{Pg8n$M@N)>l2yoRG!JbnXYDcW}ruNn+!L7
z1ZAwsf>_{NGu_O-S#DOZELS_&xrd2kIqhyB@zfv5at+N{Za&x}yR+PE(t3*DpTbMa
z$B;fe2@5Be;g$@}bVCU<3O9{3hw~hRYn+(r@_Ftek4<007Z<0ySNXk!Hok}F?BUM6
zNciRO$rhgL2WGli#5;2VJlmB#H8{5ueELf2X0LONi7fXf?{neJ=?fF?b>e7lp8=0#
zxEioobqQC$BjKt@a}RV_LtbaY8-vNy4{^<4$>Z<`{-)O?+>AvD(nz@1h;!!3gqt<S
zxqA4Zfqaz^hj@QB*z^w0y^WtohB>$Hh;!5U_62b-=>^~PB)(AzR|mGdFZ}X6{6e_@
z!T%gE@#6dZUehk&KH=H)3EwYDa|`&kgmTwrIJb!3f8kpr&lQ_!OA1H2n|L<;8~+SS
zJNUgvcwM?Xjk44nPIrGJ%&a5n?suZ)Qs~R~Z@^aZzE^%1(v}aTyV*PVb|HDY0H^kR
z8Eu!obQZjj90hGhJNFggW>CJ_@WEli9>BdtJT=qsGm-Xm0dcIO{ZC7G?|?5kE!`c3
zo;AG_Zv9E;meZ#+4o-K6!Isb`o#c5Oca(UR^PYqbHC>%s&+q!)&OJ#PmM;J!?j-5Y
z;JM^D@8qem#<}B!Ti-6-H9&{jfuv76tfjs*W+mIHuXFAsewyCSa!vHJ8C~Fu2K?QU
z>6+WX?-O`d^1Lp~J>9#LYg&=znpbDJnbhU1QJvuTPOcqwI6cwc&EP#r-PO={JWswJ
zp^j%Cb?z~KAD|2mk+1raPOgElod~m0;d(kZT=f9Ge!|mtqo2X`#C65>#%17=y(kmz
z--PKv9&4)^^LW+_hj$hy+&SP&PLjWV)C=QmSAByX%|B0rmpZr$`97NXSG>{5&Fb&m
zpP+ri;!Z9deC}z^{gQ9*fi2|yK7J>+cA{N(a(l`5j5(d$Kfrq`UHGgw&u+Z;<@r6H
zHH(O|2lUt^`jGx7q`9#@{cd}=teHC9LmBb=NKGd<yP=a)yS<EX1HlIHT>l1PiFXw5
zLvgdHr!hPyl8>wSz9fUZoS;rtq`Q3Zo4{rdqn?k^zIdOEzw5!o2h(`3!Zodi&t7-#
z0A-q0=-fe`H9Kg7(Cq_ZTNCb9!pvQ2{IrNLBYCeUA4~cD^jyYG-ao>x+WsH-{TtjX
zxF+0;p~OWTHMr!`gd0d5&7A1m{d}t>Zx4Cr9;6%%(7GPKkC6YdeA|YbO&ZJTvzHA^
zxP0=yjyx?H&DgLKp2XG2QI;gGabm(1;paKUI|n+F&QRPE=skw#C|o|SrUd`E^%?LU
zY4!!Hqa4MATS0mY_&o>L*bX|bBj2>;ByL7C?G1brZU?SuJN*RXbOZfCQy1|v^1&YJ
zly=iJ09U~8A!%;LaQa^S&)QGj^(Jk~SOXtTC#`bQ+{?FS+yTW2Pu4?^`a_mS$rw$w
z#B*E1Eg72bYWp*W<L{{azE8c~l;%pv{}E_Eg}CbYy$TwrjK}fw3$P{cP}ks9xTg9x
zuBM8(aW(q3iTGU`*E}-Koy7kvuv*&59G)5Qa|c}0+Z|og#gxZoxaNB@+-IcqIj-r+
zj*N4JZ-SQx;0L*(`5B(?c5=;A3~TO(pTpEUZU$~T?p?yx@{|lPnD`r(GKTDjes5&C
z{=ARC4O6;TCfsrSCJ&I;BhKBR{_-SwZ|41kXqe>+@t2Kjpo~oyC0sSXC*lr~e}(@u
z&*T{V=EFA$@{rAMcrQ83xjy(i4_du|AIT;c^ZRn#XmI+?<Zk4mACu<vu98t0zt7F0
zFQh+%pJw%<Kb)K8z9fGS7Q##WGhIE;hT~ao7-1e^?4G@!@tpX+#!oNe`xno-i-~72
z@!-^s-yn_Y#Pbf%{kSCykPqszoMeKgN2&LN8E*X%@(!Ix5${iN*|>4I{}A8O{495f
za7|m%-HK6J?g+n2_;!qEg1)*P?u34WEoo%lz;kxbOqZqSaOYN%ujwVw1UX=O0{H|w
zEe4y#bE}@GAs;j{-tfEWHu8gvS3|ue-+-1p)&D0?Qg?e3ZXteVGhQ_Gb*_T=1CK%b
znZ2PGSnV|CBIIX!9bx&M)Yw9QaV}Ws{|`A=O}Gber{L;4Ku_v<IXpj?xE})R2R)bc
zOt?Wj*Nx^oxZ2Tfo{c`?9s%z|d^5r7@ZW>qy>QE^n`8&lg9m2yfu6&tvvX~1?$7s*
zgqw^1#u3o{UHJIxgj+G#xxsvEY0ouB)7=yJS%jNS-j2dIOZZ*0hxSifU&-(3{2oS_
z+BebzkIW>k(Y%YsNzxd>?^)T*sdyIRJo#2%-Mrf51C7%ih^I5|4BTw!wu$G<xb=G}
zC(mlay+OG3;_=H8ZV2(-jC)6MwMn?&;%6d$mM?YgkNjRThccZ>-?Wu_#x1X7zKH+n
z`Ml##Y5ZMreb&V_|C{k2Y)zwc`w2Uj-%YPDuD^}^xdVDVn&tivyoUO3I;X2^z97wM
z-t<?n<}Hl<t25nV!W@>LUeK7|D{)JX;{Q7QBMWWieK)SA9sJKT;BTk774!jTk}sv9
z{<P_S(#^|s$B3&7?j8Kj?UisR6?b!Yx1aZa^7{zSLyG4${FB!jXrA29{FQIt!fEW8
zF%+KVsWE8gn1nk9t!jB5;n~cbvgr%@+(Z}GP@QlI`pHK|F~(#k+zGzd9O&k1e0R5y
zG-}3lce87{yIF&~Q-12LfU#kCcUQBtyX#3>Ggk5)zq8-z<}TuW)~4=^O?+R7gHAJc
z@LtF7hVJed@k!2lnld-={RHnCQ!}XxvXV@8pngs=zvuUo{oUP?>TYfcb=RKnjeWbj
z<=Nfba?)E&_*(of=eY#@`=r}Q{IwIodUSW|8F1H=<^sMSqCBS){w3A-M(TTA7k3-@
zddhPL&u75TrTnMidf|HDl3OW1Zh8;sh|E?CUHX8}Y@jW{6ZN#=h9e2LpST}73Ed9U
z&y$~a_<7_2VW9b3`j8s@Cgm3M8+U@V{)-b0PNR$s3li?>8oo6X5GP$r^B1&>!{{jZ
zUUQUoyHx!m?eiYy;a$;L5Pro8=X&t{yZpX~GUVWT@h<tR56>mEn;v}YuQ)~!{wu=2
zop8OuBtMM+zk=U`d0s1waHIJ>3^(f-^89|B<oda^Td+#vpM8Qhk9<FCD>%RV6Rs%_
zJpt{0Hhv2g_8oKqbNPNC)6G4Of7iw}e@4HMPkEr#j|ejne^ZHXsKTEEE&EgFVAZ&$
zdx)#2bC*z`8h<78FX^A|`tW`wZZUqAk0xGp1ogC~B_q<^O0Y4+HKU}1TfWG-WsTrG
z*LT1l<KtewE#Cx=dqZi`f7I;Fa=+GZzW<)*V91}#v$4UsLY~QH{7{cgLo~j3bG65r
z8_EA&yx&57(za_(QqRO$All%!lxI`hZf^E5`UK>mX5^s}#M?j{sL9B5!}tyRCP(86
zaWklgWDmYk{-kKY@BZKoq67J=VSFtk&6<<M2i+Hf&&93BC42B4zFb1RZRLG8?hRZM
z!ejGwY3`*i>8^^fvzS}&2Y(0G75X>Vi<g*p9LNam>MYUuV46F`_j+hjGn~BOMiBN0
z*quD*_Rn%x<F9^CmRrjEgUBq$`1UmUpLICPO(f4XjNeJz5AZkr1iA$D0g{1}=n%vU
zGY@6DpAoheUTnbsgN*qPEku5(=6zA7YlL=>j7EMa$#N%%yD3ROb77iWhrBb8IBN*g
zgZx+XtR<f09Oy+IH^E!YpR{oazC8%OWLTDalxI8Mm+Z)Lr|~?WY<x`iPUZRh(k%Da
zHu%kOq(D2$hCUsTk@@{P-P-xc7mbB@?hSnh(x%5y$H?Mp|62&R8aIpI^|bAVO$qnl
z4#wYJ-fbtouB!hjwBLQ`3E&6GTpyB`=_d()u5;h!`z-k80{Od=c5zu7_c7mpRp{M!
zz-BYPG^2lMqW!NQmT*1DhV&8A13kOYy9(5FH-gn3r`#72{^B<7*I;YuzgJA7f1(_#
z3KQ;j%CHf6VN);oqg}$K5%;H=F8Mdyix~;GVPL{-CypH_(R=K7t{-url6G5h+i(L&
z;|AQ!_Reisl<r<^NOv3XqxpbHQ;a*EG(IH0k8yqZ)=)=V`!LJxTAc1ONbgT74@7PL
zusyun-Zdas)Q@7`15Z4NtJ{QJkDIjs9bgUmUh>`smxX)sV)VAu$1>(;&9mYC6`8KC
z5Zx_U9~?iEO<$zDES`OF?~vB|1<v&(>;|4M;`;KtAMQHbHMmP~Rrp;-etYtKlKgk=
zfzD`Dy1RjI=knX*vM+!Z#m>#o@8BMDY3>>7dC}5@d#Ztc8u#Qv+9A)T3mA9Xkhkl2
z-oq23HKU7d+Mea=lu&{>X$SPg#J7zwbs5f0<o9UY&N_Ib7rKC+om@We(|dGsb-kS1
zNIG@&6(wLBm<PXD*U7DaBjMJeJ9vq?@n(2u6K*4}CZ9eCHzOOq)V!LBaxHw2JVd$&
zI=L$17=vqi9X^3^>*y2fJYxc7?2l{u<XqRh@m%-(n1s8L@;^5`jrnpX_w1;ITTJ)|
zsMAttbPet(ZLXN-ah{WT9^<*5@AEn^|K#@xT<zZWuI@-jH=oLxb)ci$1h$ZMgdIHh
z;MU^?lI~WXzp6~Q-8|=>L&9^IE7f##GwV9K1+(FahK}xuA&e=|VdWM_<<icN(r@E`
z7~T)_tQ+0Q&1gVozp{g?8QPH#9o-S|rB_0~Y-sVtnXW4$$JW=-+0m$*nKOp@)NXhL
zeAlBHfzIncgOQIDKloy3wFzG9&G(n6(<Ei6p*&3&(Vi(svYGc29Ya0Mp#G9o)WLe5
z$9V2dxRr!Q3`we<*Yi6`oono$KFzs#e4l@ayg{#pd_z1*O1C5(Q!jqcgU9CgMz6p(
z`RNnI)5v>&{f0ND<F~G_BR#?$LO$l(BHUnTsx-5C&LB@=T<QgXenDTdF5zy0W@Gp^
z8n;Bhi7y|Qr0=dFtrC7$>7Ddzx{?k&lFfTFV{akPiMVQ9Ju?1M;t2Dz?a05(3w}&|
zGm!%p@jXc!t~m*v6)*X$8Ae`d^Ud!=1D;FqvlxfymTYGHoln0qhxnSwXH$D<wUw}>
zF@yeN-co)y631TRSj2NBV+4M0Cq30w@(tW3@ET|WKBKS=dN}HNBY9Aq1JIq|cV40m
z{Rg_Huc$xDs(j67Onrwu?8PBoGG0TcE~F2?Nr$$Q@S=4R`A+k+<^p(VK5=wzhd%Wz
z*K}7$=K1i&nH}ioPk}$sV=*ymI(RmY_kO3k=C9fY{u)Idk|P*XafdD-Uys5+)YTDw
zH|-;y&ydN5J8;r$W?iWnokH_S;x8cWV&qG{XVcc^FQjjI<r-ed*M7=PUYqmqHwAw`
z2Wy0vadr9B;Ua#I0K?5cM8AZrufAd49_I0cKTMqUJ&YfZD*bb)gMGxe(ZToR7nxxp
zw5e+c#`8FSPVzj!GXvKF=W*?D30xyIJB{bLxTY<%7i13&sM5KJ|GR=Kt~&T2f!~#)
z3uAu`1DwjOd2u`Ny3y!b@p}RXQ9?QAsJ&A+wMWoV@q6ZB=BN5jU5bAf>3dh_hJh{B
zZ|FNh-$(Ji#L}NPlzPXn+E#zQ&Bb46f5f{m3O^KlaTI>I!XJTt$KYM!TPa-m$Bp9K
zKz%RaJ2Vxq%vhRm9h85{tMpIfcLuHp{+eEg9`C2Q)sz|WJ{jb^{;Xl~n<-ZE9o&B5
zltJY=#P6Ppe-H0_q5mt4^~ga<e%FB25q8EK^kc-oR&*4uype8iu#<$ZQFt)%bO!Y{
zZz1L1${bsDnvdUsv?u(`8-tEm82*|kz~_}2Zhm5bo6EOqT=PZn2+uh<)zbo=({O!B
zr<a`Yo|dPkuPNtiv_ab5i{z_@aXmSQ_t&5i?YV9Q{)S0zfnP7i^@Hc|n=BmUYH-7d
zZz!(0-vHM%a)7IOXMmfxXpn2V3BRHtefw~}&)+)8-KBCbCLC@$u71t{H;Qi><3c~y
zgS7Tu>lRS9<WcBw41LT})=+t`T}fFlBu?@%ocH<@gIwLQYuzY*55v{%xfZ#M@11d9
zX&j)Q;f3byjPvOFn+|HcC+`J3S&P+Ny?F{aI{#*L|4pCpd>#3B__c1vN$QQY)0!h`
zZUk|U#w9Zby1Hr13&?M8=%+QmUJPM1jOj_n2+?nPGf!wVU1JP<mK=>^tPk~193lVE
zZ#w0YPB47$#qWvGm$t0_K;h@$SADv8tAXEhaT7^5tkHZpkaE!9d_tJ^#LpNd{l;{7
zWyaxaT|4}qhC3J6hcMqJk5xPe>YaQubxvLkmZY3DgRgbdcrWC84bOa@$)4c7u66U~
zLWgP2eFwkuY5z_8s7v%dCHR>)ioA(W$=@dGbQ+laG!jSX_ZM5<#mBYO-zIR?;|`wn
zlzsX;gx!yyW*mH20}a$(_wao^ZYyp#Zr*zMpE%y&cW7rm^4}Z(1JS=Jzv2_()A%=|
zhTm@wbbT+N{i5s97*>Z|KNo+OQjUc@8_<IsP<+IpaV3<agf%UtNw~SB`6#?L@4z+g
zu|ylUfPBwiaE&|4_s1!h`jUB+^$q+U2UGifhu`y$GoRxb^i~JR&tdumz8{gFBhJzx
zJ^9~*zhlCoUo*c?MDgt9_xjx#ZXM(44t~GLbJP9|x3QVAn!KwI*~(bGnBNO<FGKrg
z+}V^ZK^z%8|Hyk&eVRL!-^~XH!Rzom?Pdb_!??lV({M@3+59fwKOE#{@ZPkY-&+RJ
zu`w1SBTOfqnT5^`;#(bKzW5vUIPxcS9dNCyr+gZJ5c`t#(Bm4OlX2qT1w7TiJ<W6G
zA=)tI|DnE-kHva+W*$bJ)%Sw0=zoR_<EeZu=XYIi<|NOOpJMW}m;6xv=?P>EWR|8p
z^3s&%hD7l-wI@GqX{)CZ*E4)yN4SmjKd<q8g=f;r)rfzsO*B18KXDuQ1>j1fz4{H@
z6kHq1HQG6L(vTnM_FBT-%<nz8-zof=gl|h5eVzQP-ogGP{37&(Jf#DkNx70;nKw35
zK6s*O05Vr!;zBl_iG0Ualw>`9!c3mahh(@Mo(~<xFHYk~eNW=ZC)@<eRY`i6@nl|l
z5B_H3G!EUzvk6`MR6Xer)fdeqoaSk5!DiN8=3b+&ljsFzp3HJj${%A<J#FTf{4T=j
z_w&MUB7WM)ZTwz^d*YQ0_c&w9f^(P{Q~&esVopq&_2lW%uLro9(C3#cGF<IY^d}1c
zZkl^)BYwxQX1hAWE&r^It5;p@BVNYOCEMG$b@F$AntPRJDeiS#W7{_FEuJrY)rR#N
z@|NM;yS%^jNxIwIrH$Kk5jgX>4Ll#JCjDuwtyi{jGvT$J&okes0_*DBdxY6`3-!)i
zVavbakrl)R9k+vPZEBWez<y~ipZBIVY3?EN^a$T4^8U!V<dc4)lHYTfyVn!vldNMe
zyck^=7_vvQ>6SF;LHrT@9#cF~emCg1aN^s{?^(y$m+&rnEyh!(A<6xe`43DN_u~GH
zo8K3D_CZ%&kmlHql-y6;^C|cL@|*VT=F=W->x|CsEavfNyW}5uDvqo9y=h;%+sNEm
zaqPb<&D{;Au=__62kH0W{a__}jn^r=hc-M9Ceb^B*E~yIf0pK+e;vK`J$%RiLiw*v
zlm5dk;`j4kr@4b)p|@#DcMINT4H9g_!F0EbFw4oqWBZ)T1KYXVxgwrV@GR$9jZ5Ox
zewwyl>vkfrKg9d?Dec`3esBA#y?c3Nd$)z>W8j-{FKub>w(;D=MwAzyMMv>$d$*YP
z_1C4lHSqmf+W3|X)-P{jod|p_Zp9Gj3Ju=l_i~=kL&K%@@Jiow_tL<0_Za!y#B<>o
z=K8~#^V7zjDntjs^BLT)>Jsjk%<X19lje3dI`{Ze=bqR>{a=aRZT8i!X-hxGw|?$d
z3)oKtUoFD#Gsn@ntWUT#Q_!=K?oQmof%M-zpN4myYDai}{}FfarZmU?Ik)<rG}j0n
zzXAIP?mxIsaI@}Dcc1fIOMUI6eLMl()6OB>>NJ<hbLQ9RR6a|02REj<j=ZlWzBP|h
zCj;<LeVqdK1Zf`JM?X@V?j9$OU9P=bRN~w-+3c0$`+E3k!)@#x!T(u`lm74^blCvk
zYkd6{Sj+eC@cTSmH=NR(Ih*upLrQA_?f!Av-1qpl@e}AuoXYRz^80R@+xQiEQeK`-
zbC*QeLc+hqH}rGvDzG2m9)u5Hx{~$=JqPiAJ#GWvZ;YNl<@pRWTZ9|V?~ypIq37@{
z!BygZhWphp##5dRxUslLaZlj(<9?6(6n7B!OWX`(p_&s3x0>G@aNBUZaj)Z+;*uMU
zZw|GEzU1o|7<;zd9P0WwzYjTh9Uk~#erKoo#}f_65NFblOrei;ZQPANVEqbrEADpO
z9k@l%>hswJ?wQ32cQ^0%;+|VbIe3o6jmMSZmN(F5cuvBt9F}lXd9GgR+%I@e$IZlT
z=t&#q`7rJ=+)K>&Hupz=)F<H{=lv<%))UA9r=dSO7oO(*IovYbDqM0MV^2M0<M#~4
zsoIhBmG`u9FY;c?dU)~x<7sc`Nk6m`-r5Gf6E};tH1nA@ZUbZQtGxe4IOW~hCBtn-
zCfKw;-MtC+GGp+zW9jaxyZ)G{J&4{A8D%Hmb`bA&+)j6nTT0tq)*;=!h5y<v=eR%c
z{2$yB!mVZ;Z^W&^&)Rc&fByWm#`n)lTSdAr^d$T(=enIYo#)=e|4uZ(8xNkBw&~rj
zX`3(Wmi9Sy_P^kVo<-l;@4U1R=(Dyt<jTA*X*<8_lD5lrPMh22kBLV={72&8kc4}z
z?Vz;3;Qu4sr?|i2cCPN?zU282+&8%Y;L@0$XW~x5)vPOli0J3K-0r6Do9yZ?c*@m2
zy3@^S`+M5RN3Nmx>O}p)ixUr>^NYlT`_4$rIk+@2`-{61kNo?uiHGlcB(ZMq*=~KJ
zz`Z!92pw#Z+c>PqZCX&~IuiH62?=*P&$Dpn;kx0Hv?J-0Li^<Xd$`3*)7*3D!<P)s
za?i7OZYi6!ma*Y#Ig9)&5?$O%1j|*!y1N$=o!#o=XSl}3Gu)cQnQrZgv)zh5nQkR+
z)$mOB!q7~&nr9<(_BC@d-P$_z`AOROIqB{a(zzUW748SPLAdL2ThQZgt;%xSre(R8
zd2Z)@2YQ8_tPSkKty`JN`dp@au{z6bz->gaya~5?OqP3T6i?{6VK;kSiR(MK&0RCy
z%{*_#Z35p`nCV_7U$^sq2X5y|;>B&p?Z7P|?z_S7#pU3};+Bnu2H@lQ{X&AW_0Dvq
z{4U3>J(}g#;2LqOaVtn;W$%PrMLT+7ApHQ(M&8%(TuZy2M3||#U*M+WX6m_vI>04y
zTNa^DZ-l>=Cfv*Pt=p^MuW7Uup1XKoM?bOtBy;86j19Prn`oOmXrnw|BHV+-_b~1;
z+y?rC$9Znzxf%Bqzh}Z*vydydk++xoqR*!d@4)SxmT<eO(EA^yP2=x5+%nuMJ;9dZ
z*6{m<YRWc|^!dFRx3)hxeM93Y%8J_twiEX%?l-tMafenX+*>?9xIfeVf#?6=-ot%9
zGSmGp&%fYyZamL@#Pd_!-*7Vvy0}^N>0k2uAGmLD|G~BBLBy|V186h2j=0ltXW>?K
zWt<+I?pEPmz^%qL4x_zKq`l*6FX-yd!(TVtw{YLVeGhjD?sD9gBlK;@o!fTOxtEVY
z|GCf~x0Ake7jE4l^t8AaaT{<Oahr}Jf9+?i!F|wPIy>BD^x<u@+*QQ+1Kc3o^|%{x
zH{))_-H!YG$t-sV&%5=z9-eMUbNBK)2e&hMo*T<^GuS5Fcz(Z}4~=o9{4U4sLdMw1
z`wr@EJMN`{nQk*~6K-RVOt+ygZ3VX;zFpTY)9r$ncaF$(J9uuVZN6MWo2broTPJ3^
zEj+{gxsm>0sdH0_{};IFxS6;Iacl9nhBnklxYf89YG@A&;rY3lZUxUw{7k|<j9ap&
zt9y*+Ql86jkMnz`qx~aat?Jdyy?|TYzng2st>O3Dp55FAem{l3U*ev_EyHa_IkgG5
zir+8q@8-6l{942B7je4^ySts)-Q5nJ+jn<!D~@$@D{-rEFW^@9CH{f<=ef3bH@BHE
z+i*K^ui}1#dlUB-?hm*vOvJY`+1|$UWhTbknLzKDOB%RcbGo~Aqu3uXrki__=Z2Hr
z-A4Rv!vAKTFEO$If27@gaFy4c;Q51)uwgAZFop@VkptCGfemb60uvZZj@iHlwqXMs
z*uVxhW@9#{0u|VXYFHzKT8Kag5=fu}ZJ2<fJJOZz)$6@_z4rzxP%#y#h6>b}3Y3Kc
z6*C3}s-XfEsrU1U;C65K)cmo3eCzi+=XvgV&Uv2m_B(I4YTGN}s&KC125usjc_mci
zHofksx>Hz{-_xf5g*rjHlM}=-dKbGG(F@fDC|@am9W4%b{3f+myq69M_vjDs2%XcE
z-$}l?a-1Qv$c;9iw^;hcEiR6AWIcD{xH76A(XdN<tSmn9`vK&YXm>CIMHq#4d8uQ=
z(>&ItPUz;&a7TG8Cx16qtA7&8KiQH||IkyyPal@|R;xcO>>MMUaVW+_#K!CM`2LgW
zb#LW`_#1g)Dt$T{R_ZJ8OQ3#JUYN;mHs+!f^RNJmumqnh|I4^nVGZ`^yH|cAKUDlh
zept)B4i)(Yq4zUG!g_j7jWR>_{dip1Om4+?l%X7z*oT&rL&E{`5RTv&(mV7&aDv{o
zQvYM=&~S<#Lry)BMKmYct*)tAYHgD=#ib`oYkf-EGSWykB-r9L&xN?(CDDv%?Rc8p
zNk;P(?aFfpI*0qd__%(i=g@e?9PjnK&`h?x<^8LVQ%KX>-mw3m9lw4h^nMX9{F!=Q
zUdI`n$3<MmRa`^%)#2d=*;nyrA)@M}?LXX=zsnDM7-zTXu^ql2?$GO^v`OPV`U5od
z%UejG{&jKEPM+ic2v0D8^j`JR{$U}{a7veYqLX_VeFSn3)koYJ^+-2a#BUVFU>u4u
z5tGqZF+NNsd-MyZlf8e;H`wUCjepvInaQu_zPbqULUmJtItq#5>MZqPgL-hbd*))W
z{C#R@D5cNC0xUxNNOj>@aa4D9q5GkJfia9Mnfp-xW3jq)xjJ>Vx|M9@PR&r~q77Bn
zGAt3!GOWNV#P(YMu>ZNRhF)j=Vw`&|eH|Kvo8(UH(tjY=^K0qRzaTdweQQ|QN^VE$
zzIbKC%eyS&F2`WIt!#E5_72tmHRGj!($hXl9a%5k3F%0RZ-cm2`prHZz#$w#d51oE
zlW}*l{jU7CTmIWG|B>Ctm47nJog*8C*HkF|-fPP(`OkMxk!iGDQU0q7!ZBf<z$wHL
zJ6RB_Qw1SOuY0c`#2*)gN{j0DysS;0plr-hMrJ50mkL6&ds^=2g*4fTw0ZU{c?PK-
z{cUY{*S4YI9QS!-XABLw5`BMVGJ~2)%HJ$`v_$zM>!%6(p!OdPxac>RaTV84um667
zyouXrKRQ(ZOBt{BoDIU=B3xxZOXj#6?+*=4kF9MuZQKB@P3ne>aex&2vvz2>BaC}^
zfJb<O0pz_V|JIBPS&M#pemo=$<L-S~ds`^~Zjyg7f?w6cp`nNzg$nlcI5MWZRo@yK
zis^OAVSEdF`Ip|`=L$oEcbAmT#Bymx&sT?qiSFxt$^Mj|zYr$Vr(!y0Vm9WY6!WkE
zi_rI1e-^TTcrh%YFT)D7_~xs~)KTA;z9#Zlp4XC{@<P{Q`D3~Kv0DBha|?Zc1H5Bg
zUx)SBh#0@><wapLy>3?#+o33IrEf>W?xK)9TI3rQg?e(QK5jHG6|EyKbG`4=!$Ub)
ziGApOwJ;nYd%kWwC-OH|w5Qv61pOFlgi|Y=_<Kd6&cJECfw07B&p^Y&qEPWW<A?4$
zg&5*Uq8VwlpMEiPpc7r_Mh02r9=;eFtvhI%Ulf`R8n=+GtF2j(zBJlUv()ofdcOPW
zj(fiJCFn`AAyX7OJtK=VIEVANh|9Q&cJc2J|IS1aJH`9;ZZl*SQUAR0tmj|we6mH_
zTFF$(^M&0;wiFbFYr?sKo4Ad1p?h$L-Zfm<LyE#Z`UB)P6@@H!hJW|SqEPPrN8fO8
z9`sFX-+Zel?i)ZgJw9K$7F&1mm1s;Ngd}&vEa_Y>o&0N7D}QguqxSmM$*1)j<jp2|
z6b+~_KAZO?{SJ&k5k_GQ#-V+a@`u54PIg~W{;n&3WbT6UcS-w)X0)Jnsq%+3+E6Tv
zmJjp7L^8WG-}qc!m`tCF!TMulUg*Mf?wQCQ6fU}lSaX8e{N|#H4O~iAYqRH(3lQ6u
z7Z#CAP$!*nBo<2}8pJV)nk&+O-L+}#ze(D}5^W=y<Zd9B`Rxj<!Wyi_dThjIwC`5`
z?p6ODSH2GW{!P9=vfJ1nXp|qCZs~vZ=zl#{|9+_cy{>#-QU7MN{l8bXgwy*{QP@sC
zvn$K!<@)u}7}B1P?6c#p#6BEA-#3fGA@T^0VfQz7gcIZ`^nS~_2(qV@?ML?gRDLF7
z>a~hLsI%#5bfQI@vyRN-49+3VFQx50PwzUZ4b#3}q+dpk&5%uKyU?Aou3rA@v%c+F
z{>y%2Xt?UW%0I|&-!cA5zk!>$jXUW5a(=i+_I#iHUOg;4pg+PB3?Rm@TG)B7`)=C8
z$X*#nAAtsONlw#7qFx+}__Y{k7)6djdf{_n99fK%^%$M#!bI-L$j;I)m}C54zJ5Xp
z`(GHf+Q>Lrr$1AFT-f_Pqe=UVseUsZGcg;3^?#H44s*HNy`K*6Vz8YT=WhDP^{e`}
z@sjoz%~!O)uC=-+)uI1UtN$ULRheO-R5<fcQDL0)htZlA=^t$zgkK#QN9FV0=iBn+
z4sqKgj@!g_$gr@)JuM0Olw5&ytu}yMgB1VH4f3UWXf5|TWaY=)Ecx_J`E<Gdhq6Cd
z{>eJjBcTpR9#=*Vvtj*aBQ|3zwxbN?s6@MaI^5fNQaMHUCgmSl<j{Cm`G2VVqvfpf
zudJm~`U_3gu411s4&V@~ZkanFtI1f8c_R7=oI-q(_?3v?H1QiRext>2hx?*yNTT9P
zZ(FdUJ^EAsvYB7@qqjpRS^1S6#^l6{eg@}o9yQ0MU%cX{g^zlA;)3w+3m+HVa~W50
z4etg05Ar5%qy4n<k4|(c*WDMq*N4h~zxRCH_kUgaM>AT`iWJglL(dn7g*(FOwa(xk
z`2dgb1QkEt5wi7fhXH!tJasU#?eByUWZ$QTgd%bj#$X(Z(euM0VItZ4W#jQ=G<F-!
zUH9#jXXjXN&2J_iX&+{jJ&o^(i~E25c9_dO8KtOfm*0M^E~PKPA}ql&tU%vq-VUqC
zmc2W|8gebxVLj6Ocd#FJgpKsB`DLLKo9SDTqi2y>UZyWnX0OVQu$_My%2A2f;vJzH
z`{;GM%R(Fn=!eiiPa<)+EYu^ae_CdgeO&*f@4pi&%u5_`Z)$>PUw9{UU3%wfJ^Jft
zT<#t9@;mwo@8~DI6KZa~6KWs66XN~vggUbR?mHnd%QH&KLIZhBI45ulF~pHXGty{(
zzbth0cn-SIeM{Otl*XfFA&17rW%gy4g=Vy%^<h~^JuVAr_q3r(d^&}b#TlGKEL9e&
z-+U*Wr`H{p{=H@4BK<NN#49Pi3GbjDSNTQhzee6b`s|Q!le~@8CHe9_`4V@y?;$IH
z=Cn`U{4%JyzC-$VNdFGwFFQiLvXvMj|Ldnlebonk^9WBcfWdLR|4}_SQaMBWNacU5
z@;^cONB0%&@^$qXnNyA$7e;HGmH(B>KUx<n|1*?-v>|JJscO@X!8wSi{g3U=3nPSI
zgzAIJ<zD3ymF9K#yr9kA|Fm4z^Gjfq`zn5E-O;ZK*f&GN8159AzB@FGqZeZ$I@tnU
zZ|XPfP(P7bL~DX4yEYZMkRPhlG1KYQWNezT4f<@%MLZ!M)#`_g=aNZ8^<$}P^RNJm
zumruo8XA_7(KzY~vhU@gVHLRsYq1XNu@ODreJ3o4%3r=R!o5B6`|*%aM)rPvh)FB&
z8Re6d;or*d{43G-L0Q;GwhS2(4n%&xv*+$}`QZ>ft$t4x7)z!{W1^kLJde>&AgA5R
zUNZh6%x?a}_-(~0#Bg#*e)#D~?_z8Sr_Sbw8tqlBHY)z1_OV}^b64AQMH_To`>6eF
z)Q&bSf6n-tvCs|1_qQ0|A8lNJyzzaqeVX}!S=!+l+TlX&?pW<Evct8*i1zx&JvWJF
zq!HVq{hO~%rPnpFzfV3Fe(>IgkmYW;&;CBh{yxh7=70G5mhj`#e;NMv!gs@wY5#XP
zxcb}Shs(bme)RrdhC_?r3de8#Rd{#&{}cYBxSi<uEBpA~2|ro>-SBrO-wwwfza7fo
z_)hq1i_Xso`y47qZV7u(VN!GV{%_m6`<LNw&TbCpUAu?_*UQ2WX6y*xFWD6KAKw%X
zjolG`l-V9GyVm!^;o*pP^S4W_uUow>93H+c9NSb7{w`AxuDX5=H_&4}#Jk3TdY#L0
za<}rzugYS{Q`Uc14|(3cUHe;Vhlf~Zc&M!&9cs+g)FIwKIwU$qhWe(Fp~1XP^5myN
zlZ~y7x5kH-566b)A)gMZA%7lPcZ>}cFZs?G+&^(x{!vff@tn%9+N1rg5bn_*p!16F
za@}`2?K>emBR}NO?LBAEobX*!zBAcW?YkcLT_5_+^weSB=B?+9o9RcP7D+Uq9?|}a
zN1pKn1IYWbHVh+Bgi)v&SrBT+8)KML5b97rtso@4zvRM#(C~Ue7~`ICD8@ug##Btl
zOti07|GcICVcU11o2{S0pq-G?{}^tMDVlrKKWN>e{@J1aA=@_aYswE5K|N$s!ffHq
zMfy^HNLA;DQhL{gd~Ha6m`7iLT!*lmJY$w;VG+NohxuU%xeT$x`C%2g2Gu?J>aTqD
zm%6K_FjW5Pg^+w`T%+GShrUq)k^SHEh37)=m&`?c{W<HYl=tht*A?@@!dQp(*oe*8
zitXroSAFoy=Rz6194)idkz^`mPKi8#XpSlS5xbk-bx>PiTp+Vl-M?2GKxS8}|L+!t
z+Wx{2e^?mmHn0n~unTvv7sv+g6@GgJ$8Z9t5JMbEv_EA3_OnsRuGiV$$Sh=kZ(@I=
zaWwmPJo}eyxxoIt!nVE6{w3SEt9BHIX5pmKi7aBmtHv36-TQ3VOU40e^1?aphOy$t
z?oE*Or`f%Wy|3lo7h0CS7|y#V<+*8Gq*s2RACu2MU2S~8xeeX3#BYxH(R21?)GTNJ
zul9_??Bs+nQ^L-82HC)USr}Jw4L5KTw{Zve@bUPs^4x{)x4dIy^#yZR^#86%f3@@r
ztL3Eh(^K^Haq0K|tHu|F2f}%T+fi8|V<pm$0eamV;wHU$^OYSmyd`e@5?jP|TTvLn
zuLz?s2IEkSiI|K*du6ouqfb1Qdpc&K;;+O}{m}c;i(xKz&j*E}A{dizlviIe&dP5d
z7GM#UU>W*;{$f}`uEH9u#X77<i#B#68I8TCw+s!N>07ZK(HMLey5;|j{Ga_$``a@#
zl(|-pO6<b{9KsPCL(L}r^KDT-!2BTUk197v(i_mItPhUwEjGS4-}v6^#upZPzOvPZ
z_6f!((1|W|BZDk*7~H2l()&R)|Ip{1obcRJ$bQOv#4m#}6z}KnjR}y=NTU;3#Ko-+
zgX8btRBxTuFL+NsK_4h_#TqvERLz%n$@7S*uP&08QN7%`F61@bz)jr79o$1-g|VLc
z{Lu6F@(6eDOGCmV@(Bizx4=7>&xVmBP=rxP_iO(#jvm=0ov8RAKNNH4PWtW{c@*8K
zDbT);e`;gaO?zrnCrY&Mv$XG1hJ~2tl&kM|-BLGUj4&pm@@3_!(fggm{ynR$zF?f2
z?A9;HkXh~=*|?JZi)LZ9pmh%WcM|&-ZJt|od{~$)ys4OunTTby|Cmj$b3Kl^^inkR
zh+Drt2<kD9UyJxHAQvHh*LXg;3@P#L+^>DpUs%Dt3fV3C9Xs?xc4^<p8u6$Vuef;C
zp`M;Va+~z)hph3NwOEJs*oe*8itT9E-gjJ=e(~xOpYB`IuaA@^qp_04Lg^nV{b=Fe
z`cV4amnPe|tD20Vu+MfWqtP1M+1_`oR{BqxGZ22A@Z;~fho1d5`(GTBGsH29>)Byp
zAODv7)*5~L>G(i;wf-Q#Lr5+6?H8(-aD@99vL))J3BLUtd0>{c1N=^*LLHgi`ErQS
zdtUZTvNuS>^V36;-i)ZdNt2z(;tbB=JgT&*7s=}PtzjguB6cY+TqAFw?yNG5M2E7G
zQbzQrqcO(X4F#cQrMAD&b=M->KC(d?3S8%x{r329)9-KN4hGkCN8`;$&98EI^vHu#
z*dAo(hsrh?^#P)~C8}eZ7PCRn!ZvB1rtP1k4=_jDKT;onUZcF$if0^kw>-OF97Zea
z<Gt%bhxd8L6AU2le^Liy1d1>U?fTgr!@XzsclllS4DS_#_Wwimzc%4x``@=|ov&;z
zX8)6IzD4Cv$6McF4pvxw|JfOJWHBa2?q3WG72kO|^!$S|!rj|wJc^9$gXxj`%ktD%
zb_#tq=3@6ppVPK_hW<b)ch^~Q>mL&4(H9{1p17SBN3z>}RW`OilD0*zEx|IxMi+-_
zhb^t3*A<wXHXpx=z6K4$&3_LuM@`ndFIw}v3z1#8*7bE*k3FB#mb|S!pl`-jY)2W&
zQHg!1seUEYCd{#ub>`gb$pm+jY#9AYIN+W`ID%t1fm4Vfj`l+H|6|SnPcZ+FZgx#%
z+hh&Y<W3uZ-Q<h_G^1s{`Ty6=|1UQGUt<1$7C&nYlEP_58l92kmqM1_m9l;TXXxjU
zGq<1JT^uq8i$iy{a|5Ore}Bt+(7r}xI7UD3zKf`)KU>#unY-cx`O2Dyh6~<Zk9T;*
z`Ul^j_LgzlyT+f%dfzHRCb=8P>`&zp-{-2|Uqj`ml=HtF5^m6MqJ2j+{=~*Z7rK#=
z_UwJ>N8@q%Z@%#-d9H<QT`g~rY3?@iwlMDC9v<Kk`o54C`aUWSPv||rA0K)@$P2xn
zFAhBwL&5;Ryssz+7=adL=*Pyyis+*-22r_+^4~amm-5vqj~3G>BFCM5-xz`N)=f_4
zHx<)S^`X2>RzGHc(`VD?q7?Bf%BT9Ee)Q8iVsQMAd*>V8JEHtI&wYzf5%TqI^vUU!
zKi8l6Dtny10;{kFYq1VB3FSYd{HLPuz5mhP|9J18jLLsB|FPbE8?hN%u^nY7N4tAF
zrYU2Sl>ZA)%T(sN@1}g^l(WVi-v4s%f3^2dwl4PmOT7PC-oM}Ve7ZQAo7vg>#p1A!
zJb*(uf@3&=Q^<ZZKg7sgTeK7C`{0#9n=Z~TiNXHI+w2^AGzQp7_Wo2pAkW|&`dULc
zPhP}jTt%wD+B-5GLBB@cK#O_dn`CCJHci`=dqbN>-sX1)UBk6`6SQCb1>qj|13bbL
z#8zv+FhH+csx8@OEPtUqgTeM|q4w)_?H5_&dhJU02)_>XNFa#@jPRQxjKUZ^9e=lf
zsY(BayL~acb-8wawRWED=FVv2v)cCDCU)ytcIyRpEZK6M-Abmo(`4H=ejh#;iiI-~
z6+idg{<Ck>;~xH#`3<IXi1GLDDR;(ma?+jU&XCc2nd+XZ#B*UfITNvJYskpCsOBG^
zFI|g`@pI2Zk~^B$uU{^mWR0}fzA611+(-U@(;uz>S>ShzumsDn0;{kF?FZ#QZDFT=
zT^G7HsoRl74vqKaKQup-|K!Wov+`e)G@=bHx9lkq&N{5eMx=WRq`x3+rgtgFo!Cm>
zj@%aIcZl*kT=|`l7s~imSuapdRw5?f>?04LI>D~rq+H<;clNXT?en$mGnC^umE*<A
z?-k`YvUhuoQMo5t|8s(j*8iL$qq-<Y4z54=w*J4mCeEEiGg|iR>yw?R*!e=plIaW^
zV3&2t^e%QmC(hB&BS+67gKpHkuKbHfT)d*SGW83^YlpsvbT)KE{g1+M(eEzfDk{G7
zLbyiWz)iF(&mGEjC!)1}-O77rwD&*K`$yw@#y`=lu4_RnJ%zOU+O{bFf5rCy2|KLa
zIKVfQFJa!nJv_i8Ji!3+7KNQ3Nq^rr@H=ILJ8FBQI<RWDHTz`sUi$&~k3uZPF8Pvm
z2fuhB)Jb2I#suoofaqJ)>}UTT{FD7#%cf+XCfKJ*vVrYAOgLjO4#k*=$(V}iXy0P~
zeTVsXvdeh_-Dla?7ueTiuAluc%qF&UGg@}Df7!h$c6u6ZXkq`(6wYkSMJdwL+=F@a
zu1&@kHWY;g^hL-W7WPW>-%HJZvxArLtKwcpu0V`^yoy|dYI=MMyMC6leYw{m>3I#8
zUJUh<q;s_S?~L?cav$oZsQ>4v|B<9O;FI?AdcWU@&B*?xGC_X4{$`i&kM7;R|9;<J
z*^0*58&@iSXucbjHRZ2g`6HjLzu7K~XwEKL^IJx5S!hn5to)+-<~PEaAuRF8Ob|BN
zCG2Re&k^~d(tTB9i^4wg01n{@Vvnu=y>I<6y-t~r;{^Q_8s3n$ZPK_&TFDr{zTdnU
z;$#xdNTU;3oWVJq$3<L5@9&DjRkA1AV8(8a<^cxn=P&8+^S^<c$VPp@sQvx2bvE4h
z@BokS1Ow>%^SqGvKWiH>0!0{wF&Kxa{VgUZVlsL^vKEo-`Jz5B*|L0Sm`TpYT$Ca`
zUirs7de?B}f6mabfW8PHum5|juQXEm-=+L3Gx6iff0MFtMH#uSY>*8X)K3@e4;e0f
z#>1ER{W7dV_PYgP4cYG7cA)bu-v-^@X$Dz(PJPg*Z_%`0`QNSlZ}9zH`*{7Idn!NU
z{XDM@7S=kf$3|>MRL-OH_t?z0mAlvb-cFXG9F?eAZVr_^fI~Qf!S$Qt&7acihD+NV
zdyVL)@bUV;$G-na<^L`1527}@4)yc|lE<}Ehlhm~QFw?WiDsnHi7eXLu^o@~1@^Ol
zcWV><E<<LKOR<0NvVYyv+|O1aTe(x0*uQ8)%YC-3e)F!BdJJcx@IQ1<k1&31%w~qL
zr&y!N|2%S|`EzF`coun)UzO)n8mHNVSi+ihepgXl{ik+yi+&CF4J0>6*HY;u6K`sN
zXG#AY_wAB?)FXi;8iae(Z*Job?%@F*;Ry!NF1>^E|DN?PzQ6DL@%(?3Rt(PnXQW@)
zZQ|N8T;J#~<bRAn5z<4XpZ_R&*J$a-82UKme533p=`WCe6!WWEYz-NCTX~AjXDiaD
zqMCnPTBAPLbnclL9M>t4&ex?AyV%Il9-oTu*)z*#u55fY?D?K{?`uQDT=$n^9u{B`
zmS7oHU=@0QZ;XWO`8-=Syc*Wh*I_+2Vl#Gsp(Jc2x1+Dom^E3BO6<b{976UNuZAOJ
z`Eg|q(b!x~q4N2rJiJ5s+@gH0R6db32H1ec$i_!Ao4*CE^c2#ojsK||+BX>gGZxv2
zF5`pU{G+k^!F7mJ>`U^z6F7x9dcJAj-9HQsy<heW?q>A98u|VE(DO(2Hg_koID^V>
zy%zR-+g|@KOtAi2KZD=1^@Zo@7jYS#`Wap5Mh02raMiVIxPhCvjXSu9zCW-(zWr+G
zdDR*j?r8ntBk~Cbkhj>kA3Zb-BS)YJqY#_p43`<^Kk0SqusE{cDh<WlQJt1Vw8o(x
zHRDS|?WEEWpH>>`$oi7fkRX%X4P?x7${*U}@2r5G7$b~{7+jb1uJ`M{jv1w)6J6-O
zRLZt14O!$al!nIDrJ-p;X=t8PYOg?PXq{3TQb><24Q-?ORhNdz!kdcen29vMRH`)0
zrgz;E_R-QXmtKn8U11}GZp`CX)l(W4kc$wbFCmwq`o45smo8^Vtl(aSWUX|e9tr%Q
zU{Yw2&NZ&BMS4nUSVyi$6wl5b(lx&{Y~<dIta#-zo-JJ&@;~nQVmR^UoN)a97sI<t
zzYzXz<*aaQ>8x<{{^!F_3TK4BeRD=Qa{BY(@XBf7$BU<hAN71L9D09BIJk03_~HG}
zhSut_A$9%JVXLq!o@cKg937g5j2@iJYsrj#+W$x>o7F?ZcK1~2<DD8`5>DD!`qNFX
zhM%3aR`>P%5HoN1pL&Le*}m7whx*PR8r!*UKFyx5#MzM{`Mx>BIiCtOZ$2Mt%`?Uw
zhE_McFzlN#J^ande;3MxU5*1C|1SJs<m~YM@iW8z#WT%2PY!=w``^ny>WFXVhsu-x
zL)bHVdZ-vXHS9h(H5_}RH2j_KQ29&k*l&i0eZuJZMrn9=uQC)$!=blkhaYYG_u&Bl
zzE78iBa8n-_}h-x%-Q~jaCr6HP<y;2)a)$@bsZ%ko+$B6N<#g~5@YBk<~K`1_ILT_
zakXnWh7&l27~<$ySrXcpmY91f37v12n0qPFFDwZ;?<h+)?J5b4!fxT;ythQ#TM}9~
zm4r6#G+8y_)sPfUGt%fpY}%`#8d-YX>#u4XUkzvI=g`pM{-#%rW4@{`dNrKqcM&Z+
zhK9>zwD0jMc@5D%$Q$HMbe&ZPA=9A_N*SN@O{00l+pgWgJ@kD;T^zNA#*(7_mVMtZ
zecJy2u{1nzz4ztP@Pv%U{{|xWXY~6Y4h?xr!p=S3Q{{K^!!UYmrSiXYXc$4SW8=h8
zL?4BQNo+9nLqh#fU!uMj&LfO*Z5)a*5tA_$HQJims2$m_9oenj(B37;XZr&i`?c?{
zv#-&DR-_iPui4&hXrI77N9QE=`5gB76!!UO_W5}BdB5LG(l&X{Ow7hyJX?QQO7CJ%
zN9zyg(H9`crq1%qq=bb<{Hort4vt)g7<~n~3f1BsUoGA}*79?&MUvjYwy(c$ei}7R
z(wA}H1@~Q-J~EnHNIum5v*UK^C+*S)EI+GH_bv9+5A`#GJu1T5i1thRfA8!6_2~bh
z`=I`xb}M^W|8KAMU!SjOi~j!({eQBRJEd=)MjKkR*Xx9{8C$U(=^fgC{$=#8CSlil
zKD`pr+Py4y=A>tl`}kE285RzZhY-`3IYJ&oHUGhJ#fSR;+@~-&uDD6x@9xua#-7iz
z0r>ZRRXY(~M>EpsL>6ao4(AcAeUJ8__PxvA+!n$`ewR`4OZ{8<`x-q(rt!4@&;I<f
z{rk1r-)eb3qYRvM4Y|Ye;z;k=_l?#vCZhOO`wskS=FbVWo92WY!n=vvxPyCmfJb<O
z>|Ob2vAl3ep138i7~|<WE#J-3FS{ZidLM0t`d#u_irzd!pQ}W_bg6uc8hzth#CPdm
zpdQh>qL0@<f2S}EM9=wVVaWSm^wTf`MHq#T?f=!j|8n1Cv2Wu$5AO3I8&B&0C-ncT
zy`K*De=Ym}IQ##wIS95u&)3-fzcvQ&#bKdW-)M|5$DtS#QSn=Q0?4VDj+vN^zVG-3
zzqNj5VP2TaU5a^Fgc!eSETPv;k+w<l?N@w<-@M>mOWQPQM7?pC!S(NK{WzPyj;v=7
zC&(ms1G)2-a|Z9e7|O{7!dr$FScNrczry}!_ji(AAF}@+v;UE$=g@ea{clY`^L)1c
z>K8*RnOe;LC)<SEGKp<2oOM`_jY#uLvDr7%yLJnkjk}e;9l5>2-sKr&_rVuK8NVvy
z0Oe#QV)T9F0aP11h_9Be(=UcY+((c+`y#uJ{V%==vWDGX`@Z|;7=K6o6y*X*WjNaZ
zd(3Z6;1pumS@_geDbE;(;%;|OhkO5N{LT2gGLl6OjmlZm!#|C`sY6<?EB~o~8h=X)
zqZut7^32cW4P`pb-HB)qaOKXY`+vK{t#g9BHC7%IzuY$WkJQGiL$dC@DvzEa&ms09
z`(Isjo?fS{M`wjzq+dqE3~5^_jZ3BVjUnMGziYUGn~28UZj-%i`8#Cpv!la3vd?(O
z1M(4`U;uevRUXj$m;V2P{m}CT>k+t%FbZQZ_`bAh(fGhP?qc-)?8PvVoQj&Uf7<^$
z-~7J5dA+tOp)E~r(Z4_L{G*?Z4%xT0Q_t&Pylfx3-^|2p%tiad=!_%tA7s~lHvKN|
zA6ewkc+39ZyY~N*Eo}Z)dg_urd{^xMz2KY)Va&q<w4AnIfE*m3yRZDvmth6|+5S!6
zAv?)8@GWAs_DFoi`6}*-)+??dtB&S{wd6V^cgRPtd)LMS624h{i+sj!y=xn>8C$U(
zW$62@cSTlWA9_Dk6b_I*U$@?$jMnlFj<<NH*+y#~g?Ehqo|o*=(*B>IM`=rW-YI(5
zpWaQ39!Kt^-^sg~>)sV=v^BLU^=?MJOV%gU!`kPh_PK#P;x|b&BaKdEaR%qmu1)Vi
z=X>g1bic12M)rbw`K<AGeSoHI`UeZO^=Nf(YLa?cpP&s@>gn^sxrocSir8WM@6^}V
z=ye%m=&9$z4f;(qXup&C4T<C0aNOp12lwy*k6?dj(CrI5@q~K-c}vx$Kh^I1jJ-r3
z+@J7)^q-afE7Fg;ThiYz{ZSb~R8~f~w+N#!2IEkSiI|Lb<+(!{@4WSNY_9w8C))r0
z>VM@d`d_AIw4n8&^53tFlWl(2vf3PiaHeA>W+NJ#Pi?X0fgbHC=|m}g9&&T!y&0ad
zTRnj4Y36VEFG5uAmypX)RiGb4Cd98EgX4bcfmQq_dmnWd<VCFEUW<xfXph4`gmv^!
zzQ6U{8?hN#w()kd4CSbd_`tq@>xTMXcrNVYK7d0wf@3&=Q-~pso}cSmk-eXjC&)B9
zk;NID!+Bi9Wz-nQuSFbns7C@xG@wzv(WK63CR@}Wt@LN>kJ{8N?QHyxS^C~3(O8oH
z#}NGwbw#uXIHxXX(LP@FoNKs&n@CSk5A(11>iE#5d`Ej5?(n~doU)zemqB-vbHo<Q
zXXtyEjkel;o!@&Oe>Gm;XMA|z{zrI%0pu-{esgof$X@Sq1X+Yp7=v*r#zag;pMHMD
zN9O;(E01vZe$|*KITN!n7tx$)DLD@dun5uKiE92!=+E|VFQaE`UJn1?{o6G&>_0~w
zb*Nu$UV6Sc@x|t;$=l*xe!-dk(cyLd&e0dnDh#gw`Gfa+wm5W{&+a6<uA37lGu&A+
zx6=Om68rB9?7uIx|DJ3eZ*JZCfi%6XpI=9DSR=f(ScmmU`)$hH_(pn{`SQ-k#bGmj
zD{`)7-}j75o<(lwS0$V>vK+Bp=2OXis6HrN8R^<x91d_FLh`tDxu>2?95(+xMfwq6
z?!FEDxA5N~{bYmr`y+mH3@303F~pHXGul6t{)gh&Fa2cqU2*f=EOKZZFa5WqpMMM4
z>b?}2=58ZfuG59ni7d|G+4%_-UstD05Vmw?KWpyRwH$YLj%Up9Eb>|Yt2%5gg8WyV
zDh}t|cM&oAR^RV3y-r%<;*;1WjhmjPElCcJ;~1ZB*`*w~FGXG>Zy-(op82Mm^e%a-
z^R9e<OTK?>?CO$oK)>zU9aIP-`+a=?ZQ(uc-e3*EA72R%=sm9%hrZufL-5U4!Xxe{
z7(m|t>Rh7&eNg47j*KHws9#Dp%+gOq#diwAF!zl>5k_GQ#-SJ;AIgJ0%JySroXl`{
zlR0%kmTa10{m&$Q?-G6R3CgwpZ>zp<+xu*kk@g)5V=|_qYPbFcSxv?;lRg)vhzqyw
zminh({Uh8Y8m_R{7iw#kYqL?MPRRZw-*}9E0p|H#<p%{}4+i%as=Mn)>IW2PUkcU9
z<O26B!V>iTCWK|=3apCU-}8S2>dBsulo9UUmy5$%GP3{IMefh5zoc#VH*bga+#9hO
zY4v&vTj^b?r+w4y^fE*?X?B<PgN*iy&*oQ-O6<b{9KsPCL(LHDx)I-Eoj2RNUR#{d
zCMRq4574NsZ(@TrKi0lK)V?3pz9YR?`@V}EMs}ccgZ|l@`W!p-J;*F~j*RA3b{@6n
z(zBv9s;9^p;z**V(E9Kl-rr(t#7=6TtF_PiD1-A8+UmyL?6`yMx>|OfK1{3YDc93v
z+csyUEMy0wbEWZlbkj3<c0N(m{x*9~<!7CJ_c`q{J$+ml=%jb;C54-%pFyslj=}Q}
z_M4aBS2b22hP;T_`{p0WtEi^OAM00K(3j!9fn*1J7xnK+$65BLK4dN8`$MQZ?mRd$
zk+DxV6+**xWzTPJ;|}iO0UqHA2GDN)w*#GQ@h)`JGss?72ISAiTK!)%3$sO-t=@GC
z>6QAw$bQcJ+t>8tF#<&xg>fjxV1IL#{9WQaIPS_HvLk<I-41;!qBXbE$(fjqxfncu
zgMTT#YnpP2dGrOyjaM!at-<ZqHrHLywqucNOAxJ%Pa<(zS$ogkplWM(_^-e!tU>I!
z@h$#q>Fcl_8?hN%@yY(ncJ8vs4;9+n%D)%?&l{)YK7g!o{zK#u97B)#?gZIeGc24U
zD?akyb!6Y)izk^xGt%fpG^dv(&)^)+Bdv@_{qKwPt`g;Mmj8{TUq!Cp{fO2GbmJPo
zo<BNQiR^v#xp0%bjXSu92grWsh46$NK;ClS3M0_>lKGp}$`7h;86#4rqH~{$TpNYh
zQMO~v3!%~&LY;abo>BK4R|lbCi#iB3Z0lOY+1$}QdA)h}#8UP@8c^w(k1$4f<4}x=
zn2f1tzia*f!#}P6-)~O-u(|o;)-97c?#3DH|2gLWhnW9I>tnV((#8eau9$z{=J|;4
ze^EQlE<esLXWu934IM9r>7FqYvoRN?n1=;egmz;99Y>83oHYJ`ZfgoM#u>5;jX$7~
zjo-x9Z#G8JLbh_Jnv6f74blGlCBj*T6<CF|c&0`dg*Ei9k<vEY7zMrJr)<70(#D-}
zPd8bvtPSqJSC*px--*q4UAw)`{ng%Ie6jbs+<P=P8Lb~mayNJ<^<=`m>)p2zo3Rz!
zQHC+zc{y3*8MTOy6%Oj@i8p*7MB@`5k1xnmEf3ijzJJPheYU>jUmc&iuiPVtO3&Db
z12}{ui01c?k&!(go#_{?4>-Yn3NgfyL^E3M7lt(1i7d__y{j;!dJ4lide>6@hJ|_I
zJpCdD`wiUD99TD5<<Cqm^N+?4u8}tod%ZAJPbdsG>2(M73-;<~(C?r@o1NrN=$k})
z0H2-1Q@+@qCf9quH7fM}g}%n~|6Pdo-#y^h_o}|gr^kdx^vbXL&$q7(4NvF;7@YqX
z);}Bn<4*3@2N~}_RaPi(7=a>;!WfK0MSDSL_pUp3DC1<;aAjIuK6pMGJvYSnQ4ch!
zBbvz;_qD#|`>ghTmb3pB4++J>nTW}lis_h%zAvyTeqI!!x&NNOc+vTi)(E{@6lS|t
z9<`f0lzYtOE=Bei+TZ#1>(WzXda<z#dgYg-_lNoVZsMno=<XLkGTS45SDbTt`1x?6
z;Q4U;dSQ6C<c08eGhPVC>;*b{_fz2~C7%j^du3!e(lk08E_f;Y_}1v~qtw`NXvU|*
z!8xA}KP-GX{NRn3!&UFGh5fuFdbYN40l5rm?eYq86}k><N0C`>i~u=gvBtHv_!sjF
zyPnR!SJe2wOxM?=YEnViNLG`vY1XIFw_-cWP>xFM!vP$^5gfw_oI+J?K{$0OKb(9&
zKm7Df>((ZW4}U$&d1tQ0QF-{&VbArK%x{eiyRoluboiU5mqOCDW*jh=_5=Tc^8E{+
z^54uO!=X0{!;iKUI)_pl?D~<j!|g@P4?o`jg88lC_ORRkhst06W7y-K=)YKZZgU3f
zg#5sU4kyq07JgSX<d5Oh`|SDA{o!Y+{_vk1+8G=22W9^cq2};o{o{wmXdi||(|u#*
zzn3?EZ;bYSXzIBe8i#zOZv9<o?*B;M`p_7|hvAGc&*3~S;xew{8gAewZlh*@eyBa1
zAL6L1&JXpO{E+Z&l8^I4!~J~y?>~k+?z@Kvc!VbyK;HkR{@qq!|3`uS9|fU{>^A3^
zA+y}MyUr#*njf0bywF$`TCMd@Ax&>X)%g67{na1CFky{A5n?OzL-nluFp6HcTHG*(
zJ`N4X?Xf`u^(f}oVmy5!ITdO3&2(}mQXNn0p04H5_;-KU^YcH3ESn<77U)(+GGxtl
z>0hXAH(yq_L7&68W8$zrh(1O`hyMM3^FPW<x$-hwm~&Bzd02pU_jb6qlk7_9)1>rm
zGWs}V&T|{}VVbTOpFoT2t^A*@4`}0FB#b4fx}={&RzDmXR*<U@8$UFxA=jeLSVp`+
z+%}2haB(A(+t^#5eJ-qXz4y1yyCi!)>ny6sZLi2?ax1o@@9WOzA<I!2xqoTz2wAhZ
z-}p*@h+patb*R7JZ(giFB+-D$(Qih|w`1koOaB-Sa34a&UrHm|w+yKN2GoB8zW;#l
zKj8Zhu>S`_?%_aaJlL;)-XEILg4X2&^5}s2Z$SMwU{82|I3m2tdhH_{@)$k6sXwH)
z^@kJmuI2rq6Q}4g<c<papl8hQ_YK9HU)6Z=C7YvrX7z_O*@@~B*GEbhvfO8oyfP3P
z#Ib&ebfV_MKZaWAilc4<e<YAZ1J3!)d0fP0T*Woqz)iI8mwxf-JS_bQ=}(DUhd9<s
zKN@=ml+gj>&(ivM!1@&F7l*Vsw7IWk<v)ho!bvMncgTB@e$qXD`+(jxN!WNqe}ddd
zVNdf6bk7(VT$ffg{vZ7r!_#jR^PB<q<$YcH?+%3c8v~&Z72i_Nm@iJgC0)y<bBlD+
zN4T%|3j?8u?0LKY>HfdxhlWwyV=xZIsQ7d9|9_qzCer)9G7u({QxTorGo8$SSvmev
zL6}KTD~~DTWV7jA+UU-VIVyT7ax3K<Wi3N?YujtbD}$ri|C651Q$=fg>Xn-Y?N?G+
z&VI|>uiq`eA}ql&tiURCJeCLVD|?HTK{E5YaD^S6J(8t24Oa%0^A_c>d5CzuuMA$Y
zHi$c#pO41l)(EHf_x7)l>+nhYVLkUoY{qt!p&Wgm(|01PK4eRf&-VWvpby$7N1ZWF
zkM=Ca)dxrD$1phlCS8N+Jm`!<ey0#a97!}IjXwRzil642BmBqE$=zGw8<J;m4(HLL
z&A3SJ{?*uUnY@Z<ZaMXsZ9(s<);Dp_4f;*wjtc`B{@n@ve{-(2h3wRkPv>3hXGG)Q
z?CUAo-$~B0^PBSb?EgFcoIjy3-p+jogX4dH^zQGnuODjLwfkLUH+N>~)3!aDmu+0l
zKA*tGp2I#DZtGaKHPYAF=U40>IO^OJ;XS}3JVC6<{(%nr2k6n5e02U#-b(#2H27U~
z_CSJvbQZw~el6lvM2<q*n8O%y9HMcE!To=4TI0_>5rg{zxVv|lr`Tftf132q@{AJc
z7r()E`SU#k(f+^5elr!*F%z>f7p0ho_7DG+^dB@vaM=3C{eRm3C$7!n+CsMWOFx<B
zZX=_$iwlIa2urXGv9r=Ijl-h!dw0?PzZLvfp@BO&=jk`9C;!>`qx0ou_YCgup|7P^
zeo4LhEq#+I`hSyr^Ag{Ff;@oe{LyvpS&zZ}f8mADtNrNvz`1mv(vI-&`=NPh{pZc}
zo?jbNBDbRq<*39yRGrQX2gqvj5P1YK`Z4kZ>P{-(3HiQS`PMEsDErYF8vo7lKlbaX
zC@jR0L^IOp#Q)#%zrp=KSHuk;@Bh(vADsW0<~@1F8Jx#OT*g&g!ws~1j~)0I{cHX0
z+%5fo{rslY`u}8$@LFdnn{)L4>21EpPTyl!wegjN{=RhpH-&i{chIM8&wkGNgZBU3
z<L>>T|Kuj4@xMor`vrMQ{rQBx=gZC)_zP<l=;`-7<2u`AmGZCbcV05*Kp%lzMw*)B
zfev}$^pJ4Q{YCsoVGPEh7!xrWHN#)j2em$YEc+bw^aPT%>~l1}&%VCPzJAP>e#pK)
z%)Vy3r_q*rF|@PmJKm80(1mVfkVOtvs|){W|Lb^r)IBq{*Zy~72-E3xN7=py?Gd2Q
zMngZ__dXl=F<W?7QJBjwdT!<KoKxJbP8XL_e)EvJWn5sJae>9wE^seGc9wC0NyZ5l
z8Yh_lBKwp5&wm+KU=?ChJbwcFpI#>}@eKQ)z77pNe|o-W*Bjqh&u=3(V=K0!4CScA
zJ{-Uy9Kqmx_!Zy#y8SNPHLn~0TWV|ub#EG**<p<4pmrJ!IN_dC7;L|_;g#B}Jr&j-
z@JpiIJss#o7rL9gUt|%T_0o9X_}^pWf2WQAp_N|>X|y5QlhQ1l?02=7+VC_zJyIX_
zp>~_zRia&;<r@7Aa$D%gjMmQL9KWjL?0fPeV#}?ABd?;mNx9mjU%icO&wT^Q8TziM
zU#c&QnmOk8@LwJO^qbqbgL`;@M|gq({8z_6<@+4|TjQU3|GRbtBT$6CFZ*u{vhpX!
z?SE?=!QPzc?1`Q~*l))F<MWKCg;2~L?WvqdPR3MBM>O^l)xk6AUCMd1c3?JrE^_0Q
zKRnydQOd99tM<B(y|2or<RUD=GOWNV^ct63L$1X-RQ!Xp3Vtgu)*8!0G^Uq0EFT?}
zkFef78xd0%Y$mrN+7nXVZ_b_EN$z^6t>nKQWhh4__C>N?dHSL6Q)v8utnvR1#{adO
z_1et@nM4Eoy>XiH|Fg#b^$-5V{yNWScdbMJq4Ry~1MuwJ{_JsM0Ehj?^#h)F2uE-X
z(cZk&73cNPyCytcTX2dVLvD&Xkza=Fo+V!Vs<in@vN^hko+dj{ebDv&Pvf2CK7+w{
zOH2JG=^$%HD}TGZH`JjXiM{IoChryJ{N_9^;xew{8gAew+V#6TjJtM{UB+X&$qaXv
z%qioIC)NK6^?$YczeD}+Tm94d+acT*M!9+~TJyORcewB20itoliu%0ph`#4}W&ium
z0;k8KyiewRL;hQ+ZYM{e2&0hZm%<o&*A)40lDtbVMzm%nD?ers?N!hI=b>RD|Gq!y
z>yT41J#y;<%p{|>W;WSt{m<OUtz48wZu9-~$OTx0sQs@n{};9YOSqRIYX4V|tMFfI
z|7+9*wd$DoEcL`f^}<T^0+RFw{rAEDM;xxcWPh#^&RVQTbhc#W%P;GHdjI#m|A*fH
ze((RV_rJ^gM=s<2507jBeSi10^!(}ke=>dR>G}T~g|!)5(W1WEPNuFKKhqy5qnD!+
zozfERW$xB~XXbcs-fIrA{r2CuegM_G^?k@_PW}ja49QLM5$ZR{S8qh+Q$GVITswsr
zdMfns$lgEKw~O4~d2{4uV`P6fKBVcL=re{~@keV5e*bdFM((dX7tWC9a2^-&EFWE_
zM|+|xzhphaFJ2DUxNqPlZsQK_p~f6>ZGrvw_R-go^+W8xA8!Bs9Q*Ha)jbdJ2v0D8
zyw&PVj6nN)_TOKz|NfHw_s-t$e(e1HyUyP~>im5)PO|@gn*H~}YO&wGm7emfv}d(h
zPtdaal~5#{Q5b`9NbhwIis@09ojtFFiS)_Hy(esb8GhZKGnHS}=~u#ZawcLIUJ0|w
zxu~YctEDUPN+{)?ha|mWzjW40r)xFl?`si%Z2lkh^aPS6zMF6T$@RY#-^>dOgs}*F
zjIUI_>-@V*`hV{$V?FxU_!sMc7c2kEeb3d(KRLMn`@a4k2G{?ZgI^-NWmth#NWW!`
zz#Ha#>0Jkfy;onJz7B)yf4MWeJd0c(`THhS)&gv#S5L84g4~LjwF2A8GSrD<{Iopy
zp1epl>`=#%<*rp?9}eIUj^G$h;1ptrBZ+2oA{(L4(DPe!)aLwqfA>m=_T5Kgo4XLr
zi=1)Kv;BYP=;v_}m(lkbd5gS;8@L(y3FkI>2lwy*(fa>K<P!`aZ;kR1t)U@%%|o`R
zCq~eVFbZRk<`<m-G>$$v|2WAx!}N*B>9b_<Z2qxk78_)jehU7z^}jp&|7rdARKKq@
zmcA#yAWWxMd^bO|FZBJ<xzhJXH$8*wHuVDr*Z<b)V`aoOW$gjGCxx_Y(fZ$+!kLY^
zD8)Q1z#{bheevM_hn{c9Biy~#o-ZR;U=`LNTK~J2T!;18h-d46H`D8;Nt>~it@Q0^
z7|rfNVgfsCim?~AZ!Oz5zMTF0Ci`~>+mamIC-;Un!Eee?j!F#nU-cWK^KbWY58A(M
z;LgW?vj5q=nd6_Ze|y;fXhusv`<Km|V*5tt<hQ-g@4B_?!a0N^IELt2>fx|(g5EVr
z*r?PG+hdF{XIvsX!86EiaX;mrXbd4n#t}Oy?qoBnjUzleS0K&ZiR7E?`VG>#Njk|v
z`@h6}d!--s!W#VlBYFd}esc!ra2^+N8CP))?fcpP(j3igbXAK}M%*^}{utbU@4Yl#
zkp3vGXnjxGkak~Nir)}(zrwkR+qi@DSoe%%^U(*-9}vF>^hd}QiW4&E#uI*3^Mx0c
z4|;U=Lf-!&ZyN8dUM^kRq-(L?sIP|cPcD=$L}xD~u+u#IuJ^26yKb)ig0nZ>6P@7`
zqZiRDeegX$);^z&>|^b-vXdCEeI^^Yd*1!Gq4(3q2wtLp!Pvp4oyqBUV=xZAfA8!M
zvd8{{iR5JTea?NqofM|hqqDdZOP`MQHM}W54pAP5%d;yzZ@E6jM}=X!`(|P`vVSNH
zbIDT7!{Gl2=r3g7Fy@E$yT;?$70=Eai~b8Wb%k9heQjhTdwFm?p1XxRI^S!7aH9X=
zSVUHQ&6)b-GOWPK5A(uLM;oWH-r$t<_8CK6#eWUfVjb3F_ZKFI=q!Sb^wd$$qi?2H
z{@i(sKlZ<dyQ48m{Re5vtTwJm=D4@JrwrxMz1W8X_`&u65~A~evLCz{dcHp_9OBoz
z)A!QP*`@DNjwAezZg?g9<m@Zq@80{faBSPSaD2<S@b1|^3;*%4{<C~^@U1c7hfV(_
z{Ak0!4TmoL+i>{OzX?BnYm_r8M};F7{!RGn<-ZN(um3h|71lBA**`v19{hCJZGNxf
z@YwJ-BVYC(eJ_U-uARaUc8?ARj+*<jm+bq8$A=#+eJLE8|5Av#7RTSd;S9_z&xgbQ
zr0>U%N1OXHpO`2NN!OZ@Mklg(_rCve+)x<K(5r^zhf}-d*`xX4X9x4ce;WJ65HpWd
zJN%2GX37_pP5ZEh*mpJL^PztHtD#}a=grZ4UfuEq^Iu<3jy`Mt>kGzr?Ww!;*>Fyn
z=W!92aTV9FGwN$c{mJ)@iE`JBG`?Y6BW}$=-E!j_#tIUfjBji)zOmuOaML|^@bv#j
zoH<2*fJb<OcJufhORW(=7rNiHHeiE!b22yH`iJqxN9;dnMhjZc8Xvh}e1vS<W=+JE
z7sG&X^1i80g%L>KcaQ5u^sWM97{*ve(Z?WX>?J$I*u`jL7#PQ|>fno^n4E}M>cudb
zoQmpQ(sf+AUbl{ddnS_erAr#?mr5sUb~*nManzw62_&2RSH($ZV*1TolwuwhpuEF7
z)VJP=_F2x~EHPG~zUVqEfA5#SQ}Va=E=M+QP~PP8=K0z`?NF<GQ{%ONqpiU!G)FDG
zC1@E^7?zPKaur#j&Pm&!nEhOSSi{|=eU8@QW*&Qn{`25Eg8SM&*VbV@s^%Aljbt?$
zTU;16(<?vFUjHmV#BZrj&T8|nt8c_@@c(_pw?>?5XQ^+*>!1FA#C2_SG)B2q7~4^X
za{P)NUP<o50kn%tM?&1HrC%SvdyDp8-zrPyxEsZ>Nt~NcOFvqhq~G(?-c4IAdqEhz
z6-D6)+4B$9U63bm3NciC#@aVBiDsnH_mTWYX3_G{8H42D{KNP2*#-F_ngdNq?|J@R
zJH%~^v)SmEk(=Y&NN;BO)4Hf?RzX<7{~WI38e#|SDV<ahZqVy8p6|b4Zqjd~;j#JG
zhvr`on}5Yl^~^4mv$I?FJImZX(b~D_KZNelAK(!>7qkE8v)Na(-Ct+JBZnuh4d8ZE
z2HsExFakvwg~9%bHZuN}b`15}v%&sJ)Q%fNXuS2O^@n8iKLD*C8aqIGrS*r#C)(Nh
z9q2?CqO}JZafsF)MDr6>r;UMl&NvifB4WlLsxLYJhh8__vps(*eL5Pnxk)5YuZ{lX
z`DZg-&;EqXj@k6NsI(`3&o8Xsr_aLzEW#2jLyh*jc7ta~&%ZYQu);m7um)?f4(qWI
zo6){p|8uqeC)xEL`}ckJFS7L9Y4&du`=f*Xv6uaU)=kC+klv#Ixq+WK`mMs*jxv-Z
zt^G}*lKw3J?4ut*RHn0t+OTe9*+kKribJj)K}=acMxH=5Jw8LZlIKowSA3w~pgct9
z>er7|4w3zydfl~W|K}O)NloalC#UIOkb~`AzxrBQs`OQoQMhCb&GgD|dcR*&SI|3=
zMX&L?Gi1-tjNg!b-%u9Fi@1!dxP}|JiQey<*CKmf%@23Tdw76HsQ8TWb#efCYvoV$
zedoC_f-FMIG<}pWmxNLDaVSQbUkVfHUGFJ>%Kc>eROGm`+!?a_l5xf8{GXp&+hhM<
zbpFrB>zOV&tKa=IF&o*Ry&C3{rRaOvT!DF+dGx{b=a+;9^p;KfFl1`1?>pRH5c)E#
zK<5zo1Kr3Vi)ejP&F<1rn<`btOVy91>c`TMAd{y{LqmUQSmn2CuommE9viV4ThYGs
zHSOPP+P~Mdf3K<kUQ_?Qrv7_P{r6gE+-v_onva$`AHn{AdJ5@nuZ1@E#Jtz4Ii+E{
zu*M+vdTA&l%Tc{lycUWVD!KO|NpC<s5*VESS}K2_<=|`KfO}FMuZ8sd*TNxs<(JqE
z`TB$0U8}`!vG|pU-xTqi@LH%@Ed9$p2X$`>cZYEIN`Hsv;D|7e;RH@0hB%UFM*ADe
zA3D*6Ze)-}4viP2|BCb@I_seIr1aNHzjx7wD(^cjoK9qM2C?C#&R;AI=je4K#f|$s
z{URDRh?{sOHi_$y(r}sIRb0aj+{A6%!99FDUR@Bv1MaMTZUvsu2k_74zm&Q9S<0U>
zm?Rsx^ZrkH9wShMQ5b`9C`S8-`rGK-tAC4becueS^c)(O8-H7^tdcDsntOk&KaMoL
zt%u)g<wQ7@pVdzP%6a_s!SlN^L&H>hm#{hyT4PL~iRiz>vdEx&mvjHaOQCA4wFa)u
zMNHVG<UCZ<<5!F;UU&Wv_aa1VW*Sg`RyxVvua$<L-@N9G-BR<M&W><TRKAvxE3gVt
z`CLOr^Phd+EDdYv>(Te+*Pfm~xIzBgVQel~=k<H(c-Q);Pg$$xp3T^b?I=SzDzOg-
z(7v}MbfEL7GYk$oyP(4v2FPA=h5;H2<ex(Mhin-y|Bxx}G}-39n&a|sLLRA>e>3s~
zncz;64Lh74P@g||hVUWJI)Y<3fm4W~Mtfe1_!R9AqB-U0Y}vv2f9`eei6e<-q|u2i
z&S22~J;?sW$M&yw{#pC}@%g`Z*}DDgUov%F|KQ4>>|gf!IpLhgMO;R@!#z#b%+tH{
zB|34H{z!h!jb#&w*P#6`o>954x~1NHU7r*&&$!{<o2Y)MU437=Q27=0z<126X=@u+
zOXm*hq}K>z@chBIKGDC}CjHzExGkJJxQ7RLgeMq4-nXRxP33=s@=tc{Q2xmbca|KS
ze;UjFNAoV}M=MfDqYcr22M!a?2ozxy(vRJ9-<d!3uIti|ar9#3*!o#ymb3pckze#5
zN|VW{h|W%)PR>O1A4<{Mi8{>Yo{QuV=|a?ZNnr5*Z!T%4Q8(>hmj6<}nTP1?gGJ;L
zEW-+{!apnjyOqh~%H}R*6uH!YyZrCmVEmz4`AMk1m)k!p%v6*50c+_M<{{RRox<%p
zEd2e#pCNqY5S9J)uJ8WtTVW%)8L@uz5ZFqudtV)J$y`CS|K0p>!wz*DqCLj-r|oBS
zPZ`QliM~c-6Th&ghkgLPZyVcSi}bYHhvPi9L;Q~57*606VyLi>Ap7;gFgQLoOL|Jo
z(eh8C8H4}-_YLC!^eoQc9M0n+F5@b$;RbG^kG)?}Uugf2e8}Cae{hGqhX;6sXwCN%
zasYX6s;d#5EgYROH-bJm|2V;(0QxB89_kw+Gf7^Zrv0C0efcbN+$En_|G(S3A{siZ
z(--a-zZ-{Q>>R6{A(~s5$lbou{5}Tf*|)Jt-(s60JJviq8mq1UueJVPSS?M~|8HX3
zB08VH4b|q>v(|o36;_PBJei!1s)RXzGWsuq=syV>5^QoT5TDun>-fbZ`?<sU_1uH!
zU%#O4r$=WCrdGZf=Fz*hh#MBr7a_;~%_8HT?l;-Lo1`Cc)S>>MwFO9`0ZaU58CGBw
z)?h8xVLjTtr;h#XF|zA8`=6eP(v)KVZ;R|d=|?kK&PxAjdkc_88>$``g^j}5jIG#?
z*huRS#ukM#dbB4aj&gb>8VbY>3HL;27!KyQ(Z0uc<FT$Ez#&waL&~D^Oa2$;x7Hf)
zJAqS(A&w+!_OgEuD*uO-{|^1~EBfVU*}rIb$ZmB{8lA{SyzN{~@*K{iT{-Gdjyf^;
z{~KiHJ>~Ph@`=X%?BDs?|HayRvURogpG<SNk<t2<i^92#tGI^reEnzrf*bU%UBX`I
zKW^!_k>k!D^^Aj0`x<xnRjst=kbHpH5c8$v6I81k;)T+6c4!#j&RZwV*VO^s^<?78
zQ2(!EeE+U_5ap$>-T4LX>0z%`{KTBbubpemo&6|3j3WD<4`IyUb>l8%@3-vh(B|*?
zwspwdEg5|oGL>N8SL^fBr(!xfU)MgMTiMJk_HD@AW8?p>&Bk1mVjdP?5&Aw5kMHD%
zo?pv{+`TV5=aO83Rak?n8OA`!ia)mwm|Tz86#IWC<b{p&XZyT2)3;-A-_LvIfliwT
zB5OX>{v-O2*E-bG6NuKBMQ8O6^PBPuPuB`WYX-J*m!b00PuB;uUo!svzBZ~y`;YE}
z+NR^$sKeI3?X~`Gq4p2|?EiP~YyZ%O#vb-NnuoLB(K?p>jx@augXaqrvfuf2p_^X@
z&(8NP7ndWxZ6&!62XH9z!!gv{w|;>=A7|&+T{rLl-Y51ip7hKq<{8{~3NgfyUakF|
zpbtRrnlEi=j{K3!NE<SU#`-(?Rc+u$o<VG^{tS5@)o(r>JE$AtoI>u)NIv#nQI7;_
z{I<5q+}uLz59EpZ)xHgxlt&uKtA6vz^#RwoZ{Q}fUmFtIcbWg*Z~l9?`R}9b|Gnz=
zj5<EW{?{jMEMfomSpS0-v?BGP`ET}R8@dXVulL#d_5tqsoHAReeDQm>z9;(6oI9R%
z4;8LIAk(*$Pj$m1`rx^luiH;UA3&1cuu%D0s(c~u|MGs6g<<3f#7>%5AV(ouA5>fG
z42F~TK8kN#yc5^$fkeZ7eYJk+_}t53jC;qS7!xrWQ!yPKJB&dhI(xel(b?PG$kC(!
zI%_)Y9e5`#i@gJSYPENOHg4yAg_**bjkzerJS@N>EWs!3r)Au$um*en;GMlaUcVqe
zWPkax|L++V*79G6_1K8b*oy5aLpdt3579Y@2gpM>f@6`Nxrh_wDa0bTFymyjKQKu~
z=l?ZFZuNRPa(`=B*g5h~?eP?8?Dr1%KU-gtrJq5?cgD*f>Yt<D&0g<lsds}Ms%8|2
z74A8Q^SFrEO6UJCEe@CI(SOp%7uqLoO?b5CqTzUPi1yGYP(QyoTy;;&@K?e$@&?lM
zo8)bz{$IlGKgP=IJoNqAbZ`R4;6MWn)QkfP+?X3^-~>+KgmH{&RHJ|j3MfVe1yoSX
z4cx#D+`u(sm=5Eb8WM;hffyPX&+g2g*?WHOJ-_z^6won>(E$}yP(c}{puh=CK>;05
znB33W80w_w-v05`^So=n@7nKr-*^3Zp7o2~G1$0(VK0Q6j&CFHxJP)`D1G?C8?G52
zaLf1rve|fm7BYLo_<%9S2cYdm;|q)%=$IoOvTKCv#u)=3jvjH{73Y0i_w79-2QcVc
z+CdnKYU2`WP#6c${RjIC9DX5GE+`Jeoi`GrF&0T-RVb#{N_*<%;xK_e3H8D=j?=>H
zJfn}-zmxqw71L4P;u)joSBWs&afPuOZ<*6o-eoOt$8(Wyk^graf3#ozM=Y!B)JJJ#
z==CSf4|m>tEW~0g#WJkGDiqeAKQ8~H6J7Yd_5Z!^Ml`+1*N2ul+J+N-KT{r$KkZz{
zKZP~oT8pRa^IztlLYz0hUie0AM!9b>w?+Qnpzq3YN?jAzm@1{0p|Ef0PVI<;+7YM_
zR_Xj*<Q^o&w~su4su|W0Tlc?(zu7%GR8*@+eP0y^r-rva{IgI#Vrtk)-!*<@_`4TJ
zhC|Ljf<1djhwqOY9o`u+GVC5PI_%ppHvI6~*l^5m{qJrL2ksPyA6Zx7-B-tk{bP#5
zA@}wZWmwAZ88o5yvm?WMchv8n9UcCD{0rfT@gs*1{%Lq`%AeX7PCr`rpM-<6{v`ak
z@lV2!-u~lopyrRme!RPBYWU%-KQsUB&%)luKMOyY@xO=frzeHJ@4XPt=@;nzD{J!0
zV>$n^{f>48^401t$9bGUeE<Dlcp;pkpGNQd#i389Lf;$qSLia9U)VWZK>2%YU^4%0
zHZ6G-*U|qe{=cH)aFc!;cX1yNF@Qm@`~EQ$+vg}BhG-vtV2r%u$_M%}wabo9`=jvF
z59PmCH-yH-8=+~)8=-mpw$PIJUdWF6UYIGKBu2U>|9S14XW7s1*k94{SQKLdCgG^^
z)ZEZexh;hE;~sXS%$dITZ1{)7Cqwmk>rI~gBp>>BL+z07@>zX1q{nRvnJwSphxtyZ
zzNY<gNBaY{Q?x&3Xn)Mn{vhiePZs}FOvg;jMhWI(9@-vhf9zHd@6`TCtCQQ+&4_yx
z_f%<LEV2LFa{D5XPsiV%^$nx2{_vKfFkd_iu^3B{>n_qqQxulbI}aA|Z4`wS^i}8?
zrR_0G8)TZc$d010Mp)&XqOg`+kK}`*u#w!1DthXCk#<K>*y^|xnQg{RF47J;X`bCC
z;}2hdyl-A=i}uMy?SgyS1=3VlfAE^}{R8h0(cUU^eZ_b5Co4yG(f6QjoAeKs{$bKj
zc8!yMGL8}GAsY^Be{E84IIo#(S)^^VT>6(t{{mrmimcHoKjILMAh*pNJshKVHk#Y_
zn)#OW6nYNwZS6P5@2I(cHQtl3%4|_+B6CR2az2?y6+Jc1d%ZU}oN#;!nX}&OW$*cx
z_Z;7SyFNJWfZ7k_Q8KMesKb->KTf;m49?*KF5wEU;yMcB|JnG__gGl}<M3l2M8QrF
z*0@9;B%*Jj3DNi1;`?vC!3H6l=UAIuJhyQd4-tJ2*>*MqJ^CEt+`fvx<v;l8uu$kf
zo5p{?)Bj-5+E~6>Q;Hml<UMO`k|Pm)5YhKgi_wn9qOh;PHfcPreT41WZ9BBz%E<oz
z&exz_(D$37(EG`_?nhDR*Duij*&^#!JsXN$KLNevi7<)m`)B=9<W!XZY*=XifIUIa
zsz2f!fSL5!C_#t1q%*DlZ&&{#Pw%N>pR)fOUtyoV%sxd6J-dK?x`BO)wjJzSbf6Pm
z=tdqrxbGRw75_ZUM}^~s<YFvEwRL7|kXpySUd?7@&!@=@>X)#8cd~7#cm|_FVf^bd
z*Cd5i?Kl3OUfaw5H3okbeGTewxK>#Dp1819So2Qf|G%Ytdfq#wuNSrv*}ZK4-NyeL
z>%ZCYR^+GgOW?`)|7v!0Vf}+w-QO$XTrTdH{RiqdvB6zah6?P$9_+&b975Ys&j%gD
zJm0~d?>Nsl;rXI4|9hC{`+?_s(epj;`8ImK$klkhuB$vgJRA|vF(i?SQdpI9GxS>5
zr{eYW9O6C2y&2O9_v*I}VR@XuDHPfxGkj0V{nL)m;2g?*GkH{eQ9I;+hHy#P6;vPB
z20ttBp;o)Tu>aKzJ~H{Dp1kUu>$r*AxQqLEhyk?uw%g^unD;u-wfC`YUf6$=-oVap
zT*CfeF0YY=^||gm&U>w7XQOLt<fH#XI?CA}qu3vg;~Jq|WV5!zGkzP2;TVbBe)&^g
z9!*c(lE3a5>qjp}#&P{3_6M21W^A*tN^Oow<YXkz>PsM}qiTeH1F~%VN8g`su)jzf
z?OkJ?g_of3pX@6{#_|92$%R;qrC5fFZ)&$#zkdZit_8A+%x;taHkn&RUyJqV(AMrm
z*8|_jS?>Zp*yy)P`EWD270GV<K#*mq%4!!L)GkDY<6X#%(Jn;YH0?xkkFYpyU?17{
z>ETvz(++v#*>E5p|K-qdh&+P4GNk{D!@@Co66IfVTxWdBEcF8l{^=U^gSzwS{`5z!
zpL0X~f{qW=FDKP6m(?%F)i3CoW8Xb-ecV1xInJO77knSp>WdnrUgd{I-D~PMJQ;^p
zomT&4+2`HrKeEno`m*u-+AH<P`2ob!feJptxTjd2egda(8g1+J?+@0-O7IV;o4eG@
zv2M=4sDB>~JNOsSq@HbF&A*`j&8`$L;`o5(8EkX$oWliNLc#u@sLe%xV*g*IUq@_H
z<UOPA#>e*mO<|QAjCCOIBHI7=$%m*~`EU0Bfa5{m_G})p|9d~O|L<x4Pt^XO;XP}Y
z*OGNpyw|<|Hvi`t*9^sQjKpY^b@Q2HJKEU%?P=}*D((Mu+W(uh|Fy&NWRK(j<@}#+
zwr`dVTw_g6@fBkN%D3}jKW~2xdd2^ILw}sU?K=6z@pR0@Y?NRw<{^$hnNKdnKHuJA
zGAIAV@h3~^o!jK^P4>U0uRzZ%?{J3vy+r;-<qZBS;cKuK>yg~ce?QTjLwc<=rm}qb
z^sT6WplrG)Z||4Kk^h-JM1_~30v|8`b~)aIeet*dCOyIYa{3V*LlP-u(1aX1kjDwc
zeE`aT{YHrP_9@4GU;LwRnmmJZxPa!@&9f!1;3}>ow~~z|{3g9H|6;sxWAwXtI{(Le
zkN)@j!X9D(u^bq*&UZs5)5?gizY!|7yKnc9y5;-0=KJXNecWIlDm&^i)cL*agz};{
zL*M7LAO7)~Fg%8Ru_%lr``<M-$F~soNQ+}tMhhE@;`p0>g|^+Lp?!a;Hcn~itSaTx
zDpfv}DxXS~Po<$@b!li^R~nkOl!oRFrJ?1uQoi-lkaJ#Zwlq{PDGfC%OO4?u4YjC4
z8X44Mg8w-nZIdGOXD}I4F&)+7tU(I3sN4Id@BdBy_c!_9-wZRIGaDtCi+Pxjg;<QX
zoo{OYzp4HICja}J{O@n_f4#~7^=9Z<{-*E$P5$>c`QP8<e}9wz{Z0P&H~HV+<bN0T
z;LWg9Jj<{GtB@NZjblo~8hYmp&jxGh>(MjVvq86Wx+dC>eH6RL|HVC+;xkTq9vhvP
zH%7f``M;%Wv#_nmyzIH4@SL1mO|MzxJbLXm;huNe^Ugf*e(xyv<*B#6q)u>c87i;~
zd$13`xBd(};opt_+0Xtz%>E~v7qI`A_%@fTpUAlWO!MV!;edD!;Ruc)cWYb7B1!Lj
zjelT~_5nSE!upqcwSN!t54@~REv)k3wvZz`kgVAj^5h9r(Nm+eE5_OX*70d%&THFW
z*3KBLtxuj2b`BSC2~Wpo&I;j*<Ew~k5Zxqi<1X&wAqEiV91U8}AICpc@#~`S488wv
zjbkQ<Bl^$#zGO@?z4z<pM3G}rj0vdb`>5fsPgS|kZudo|9ml!Jv7U|l?@n^wWK6|$
z%*1Szplz7`-x2(7csjrOyz4Hj-^m8$KqH#Ac-9*{|JOW!<YsyPGlWgCW{-I0VLle(
z$@rng^w?(az%u#@{NeolkMDmS*92eXoc>?G8Tz|?zx3WuDHr|`|5VDJcr&bxVP7i^
z>&d>)m4@q{;Zk}nTQP;gxHht0y%_h3sZeIV^?Bv=PV4PBuZrAEZbgzF_wOmCm!Sf?
z@MQm<J@n3QZ3pb5A3*FA$nQ{pl3n}Ne=qWXBQ;C=08jVdUG5&(r-xi~1jmp>3K=vZ
zhqfE)e{?)h|D%iEjr?-;zxw@Y`zFEG9jARGZCSQ)-0QIQqVa!?%Eyn{A06UM))@bH
zR2=l$N7^PhK`(#bH}QeC$#HEIvhKWde&HQT$0@&`My^JCf;@+~4q;(li1NR%{@1g<
z3woS47U$k|<7xU|lYX{&oi;-pmypTI|26u5^Z{KE*CkxRRb0nS+{S-6{^+9b@2Kw&
z&1gYkjq$y{KUA{E?~3O>9%2AVwt1C4ph5qq^skmT<lkrLLs2ikWQ3(R$txHxY$Qfw
zEQ&D!lQ0=i{ZIOgr#haF*~ouIUoBZ-&4IYz_*{A%k3Wyh&S4+Sur>~TA$r~!Du1yR
z(52n-w0}dpXR+V=zMBY3$>={@My|jrtif8W$3|?%R>ZZ;OUW`+FOdIZ`G(pWY2uHF
zdlYBr^(eRge}(gQVGs7<01hGl-C?0^o%}yo9#sxJwf`LF9rut87xjOl$$8DIJ^yW<
zKbb@866^mPi`Dy=Z2p$$1N_9}@qa_uZ_2+T{^J-@$e;<g)dxAUpX~o@`SY8epY{DJ
zmB}5#tF~yvktdMcpxh--BhIBrA^kub;*M|YhBgG5^xI6F!8u&OC0xN(R4-v4zs3eV
z%0BL9bEesz``Mrm*vI$S4}00SXhJid+P97DTeJylKg>>bO|-+iUeouEC+qVy4;db=
z`=6V*jl0OrU?UIaTcCGtaxEUx2hg*G?*ZMbACI*k^d0rj0{aw_Ly_DtJPapCqUvS$
zJ<WZi{C#ajWJb7e)Qy)8R98u#G^NPecHx(WAD2GFwFb7|dhEw7^Q=aTvoQa@#e7<N
zG1@-x{t?$$={)<9edihH4|@Omz5kt_k9Yrc{WtGD>$vd#o!2~Nc$grbNtleO$h|E6
z(m0*osSN0FJd-{fJ&yD8a5vf2Xk4+d%Gb>wC+8tKczBpkE<}}drX1Hw!(zuvk-6sG
zJFdH@9ZCM%{CjpZe^%XLZEG@v`mA=lYgS+t)?h8xV<R@BO`GTcZ+(Y`583}!?0@!r
zGg|1`wEV4&--<$eT_09lV`{7T`hO#Dl4Z#A=k$GA9#o(AZnxKouszs^xBi>@-gw>4
z1pDH3>+1+Rgq|VZv#_WAm#eL@Bm5YWNFgUIyGa`9P4O5VsC--AA*&9W|3p7QFV`=6
zicIZPKHO41JcxNn`9QC~W?Zq~&LIEigG2e(28VO>3%G<UxQgqziQD-2_?f$odp~1-
zk2U`Kj6I0u*h66h7_>oK8HM$k|9Ws3N*|7q7>)inhKK%Y-_IuBqT^yrz$E0R`Nl>i
z!en~qINuL_Dt$V7dfA4Ll%eRteg8F6_-vG*!tq>k9_FKZxB6NAmO89{x~6`5q<%W9
zenLGO8u<^}`43)YOQYpA_Vp|5YsCF^pYH!NOZ|+({y#6OH^-=-5!e4;=zkVtDV8C*
z!TL8#>?=pF)eeYr6IapmKlM$jmowdu`|#_??TfU#U-SL=4x82YJ2^nI+x-6~;~`vA
z5uR5*dInSc|3v>kNBe-xBl_~I_evj9s70N$q>)jN*JG_X)?*_!<Jfq0=8Q!6Dc|<d
z5k=wY{7-G_r~AJs2cGzM8mc@W?SLk-us$)pFh9aGjq^X_+QHvHyd`WEcPYvc_wcEj
zrfjFjIS+9T<1YFh)W7E5gvC7?;~d5#Z+|lU{qe!!y$^<j!#H$wSolfzv-+;A;n+Sb
z95}im{OFYp;oVm@h5d~i!oH)Mtl_vR{NUP_u=m3)Vb}7F;qPAAs2|Pq-Tr*oqpbb@
z>fzy?_KjgTy=>Oj@HaT%_d}@IusOVSaC0ahvNh~nw>cc*fBA`e<%r*oA&Fk|RNs3|
z|Jo<VhJ8bx3qM>vG^B+0e|faIVb6pg;oWUR!~PlnDb(y99jbSZ4z-C-DXT|^^oEh4
zE;};Rw@(e3#23QRZStCHD~}e3V>S9^8^?zC@4XQI;o6^uB)$9>Z1k@^ADWz#LkIFW
zfm3KSFRbC>)X=PtvFXO-ki9o4v?M+k%D*!%oc4QRelHs?`mkd9&j>q*x7zeweq~&^
zM8AToxQ?5+jk~yyhZw-1jq)q{wwsSk_WsnEJF?%{!I9)>j72dfU=k){D&qcr)5$*m
z|C#Zak9KxEuG9ac9T0uv@&5Xs*LN&@F7j<J*aKMpzrp|FIL`5}`1SK)AwB2WXSHV*
z)8m}O4lJcFLr=SJ^WtM4WY<04-;3kI1@}MCZ!540$%ErU)!uPo4ZYTUz!cWf*Q0)m
z{%`%N=@G@DZicz$!kX8;5H^!rkz4&jC?(5~72ctLwG$PNcOgGfzr8;Fu4(%5QGHbZ
zJW{Ad9n#349(!D~4+n4vM{o>Dq|lZ&J|OFV^zC<&U5>k#7$306|18kIDUL=op&2bz
z#uwBWU!d>5b-(`q7p<GGZ5qck`dC6H#y#G5CcKGWySg}}P?-PsTXp9;>0c@Ro1`B(
z=QJNI4jp73Ipx3!@)WX-!e3P`;I!j2$j{L(P=<8P@(z?E)vpwXnmEP+@xSV}iTj}Y
zc-w#N(EhIr;hgI(;1aIjDz4)u+S1w|+7un@{0~{!f11oY?jajK(EdkLjc=t&`?Fi!
za8&yPt^181di{C*7te=|<IjgqbfFu0^q}Ew&ks%GpAXIEfwqi$UitdG^7Z-9y6*W<
zz0&(b3bm-a==~w{f%kX(g>c(F-o<@9!~h2UU-AluVmL-3+W3`=#)Z-Jsx9L}a{r((
zmR^hrcsf7-j{5hJGU1;3*S9~}Z@s@!zma`!Bno?R?ARo~Y%{h;SYiEv_lv?z`fQY7
zF6N>C9et0lYX2j(T>Y*btJ@-eWV*$l)%N)5;4t603$YmauA;D%T!s~Bo8tL<j~(7?
zCmDTu-RC|3%bx#-p8tp1|L3*;FYALOTO7x_IXBMpCwt5F|9|_1&}ZyG?>DUPFU~bs
zi}l!u&De@k+=%V|-`G##Yrb2@&HI&8<Sy*NKIGc{N27UI^e5{t7xri5%jntb-lX@*
z`pc7?Q@LnxI6@vkGWPpMiK<Qf_Ph1p>hDcC&O{v7tB>o*H2Z)1+r|c>?4bHz{n_N4
zC;bOGdI$1RE>Qn!6a3-)zv`FOzY8AM@&9i9fz{816Rtgl(>Q~3xPVKzg0|P#|LAyK
z{OH=Een)=w<Mu_vN&f#2`Txo0i`pk2XrCbG+*VXxw&t36uHz<dBYBPef6rQL^e6NG
z?$aOQ-_8HyW8V?&HRZ>E-v@nHoAJk<aa^N<Uhy;jJME95^x+tZ(HM(j6z2c!=Knv;
zzq?<%T=|?vMxL!dF8}}3IBVxk!emUvbj-wTv~BVIq2ojT|C8Eum$mK4JmOgI24zSi
znqHOv@znoBZ;fH2>>neZxtNFf$j$QoOtX#;z4I0I2Nu(pqUW$SGP==)oo|@Kq^?=!
zw-rck<IgA8plXwRm5{Hn*715|F3MM^LmKge-hnv2VWZ#k`~h3Zx2m=8yPgfD^fFXn
z7xrKu4&V@u;24rfq3@fX9U1H2CNla0b7TkdIDu0*jWeizUH(UEi~B?!J&h;%KVSE3
zIOn_zxP+(v$r}Di$8C!||JOYKmGVEj#>oG8YXASQ^8W_;AFb&7dF-##*Z)1f`!4%U
zi?jSQd=72w`me<K&(DRM!fxX(;@F@2<in_c*BY7RpiRnC#PP>N$tUBFhto%5G~zq#
zT<)Eb-P!|rGLApq?E0#0=1XrH7OvCBI=2`TP`S>0@crh4A2uJHtbft?eBqOka$LLH
zn1gH9>2o|CGcg-EVcC1bLkYd}DEs{)`;<NpJyq;?c6N6o`<<*F#r~eiPGtwzl67;~
z-(<#dJz201KWhKXcl|;vM!CM2rDU6XYDdQ`{snZ=yODpSpZ}iu2g}*t;%O3J+-tB!
z{MqsP-|4Mm^uJd<7nX^q;`7$k`<nGL=}BQ#+2_J4dTp;c^aq~{Yv^lHf6Y9E2j<S-
zF^8U9FRb~7y$Z?A$nCX9J6VeCHTNZ*ozhw6xB^e+1$G^DpQv^pHSVpj|H3Tkckk&X
z((k_G_=sJu*@JyJfI~QfV@RUyp6^fo?m#EH5chV8YdrSg>HZ5tq#rH4>VM?Ws(z|W
z8=E1X44RNb^1Su;*%2M|+DCrB=R2gIK>b8{1L@1Y4QyX7|0)B^R<pkrvAy48TN@9P
z8>6qx`Q_&CoFNPQZ}I7M3GYV!qP~7q@3j8cetmqX<r}Ee-xtT_XU?+qP_Tc$<lLYB
z=WtFu7jOwza20JEj6XmJI?;7r`KJublRe7BhAjVsIGc9xKcHoi`$O!bY(>9v=DK)p
z;x@|lf8HhU<00~27#s%3L7U|<^nF$Pp6va+xdCLQJT;OWjj<?3Qkfddz6ta=_c_ji
zo<yIF`tzP2($1+vpYN_W@Bw_1{g3UBKB`Z&`=>g8I%cB;bJ1^ahI!=tsGENf#~duA
z7v}#zJ2)(+=Y&Nc(^7ipEq%jSMqhy*&n2(C?dAvQA`A5o`?9d_YPWiwom$vmhpZp-
zi7?YOtFQ)Zu^t<-8C%iD&Td!Vb_`Sh4p#q;Q~#2A^vvLYSI0J@iA~<@+!pp?*7&y^
zT2VPg|ABbQP=Q@Y3XA*N?V-m#hf>%_KY;rE?Ef0}KR-Yn`SJ4Mkl&Bs7|Ory8~ZE$
zar6`_ex)A%n(vC<gd94MM_*A<I6?Nd3=XHr(>Q~3xPVKzf~&ZWe*H{0$=hhYWo;hv
zJ|1EKao>Y%uXT9-Z_j_XJ^(yJABvur<xTZP9E06OR!`Lak=7LXwEtrf8~;_$U)yNA
zZ}>_3Vz}!^qWlZuFa~fez3q|wFV3g)OX!95r_?Ql^@oP=KcGooZ$7F1*DlSHagV50
zvf`&-331)^VsTX((>H;vN*nKIjNT;rWaK|@jy@klE#fzu9_Je->xZdZUp79#Inxo_
zoHNPhqsGsWCCI-b?=FxpH_7*82j5%gI{tpY&BJ^w#3^}fF}W1WumY=4t)8k`^0D^0
zx+^VDXBw3c{D=+8ipI<GH`#nk{=O%FKa#)6R>y71n8NrZ=XNSz3i}U}dBpvP<Nn!e
z{Lfmf$3_(L?=@o%=uh(RR(dJw$4L7K_dwQ-*M523`$1uRrr&pA5B4F>BQF0o-{rfB
zaKLe&@%@L$BdGX>=l+*-!ZCUhDU_Gn^PFr#4*mKD^56bS=vB{mIPUvv$K(l|!f7<0
zHD;JRhYPrb++Ock-F}7M+3nlFRr+=GY!U~a>>pX(Jtx$dPn;T465?F*x@jdLy{IH)
zP`|*ud)It?e9ukCw{aKu(PmzI`{k0*L3Y0Sl5hGY{(bZ7={+yMr2ozQWi*Mq87+6r
zUq;US_SRe4j(hECE}jbI>03WD=8~Q~TVnn1k}&9dvHo#yc!oX{^)uWX;u-~Y7%r^&
z@JnGNIU2dDm%>=G7}@koq2u+JLZ@{GCODpiynE}h7C_fJ_qqC|Fj?4COvg<0ebYJ%
zWN(-gO341-&IxnLd6<udSd67uhDy)vssCVudE&y7<K6#@=84m5r8Om;<)8E1QGeY1
zdtT{Y_kW`#R7-0OQmDNqOnCe)Bi?!x`v1#ASSzme*oe*8ic+*~^ZXN@KRVHcZtDf)
zr<EAzSfc#({5N_2&TA%HUi184@%+8pR<e4h{Iy^HLT#0AS{pKb-Zy?${zjSqsK74t
zeC~Q;54jHqa0qQn<geG{>vi(iYWZuMd`{*a_mB-)_OCdbcCdfZvO)et4y}mu|Bi^~
z7?Ozl0OgjjKV}%;=UeF1Chyqb{`{6foKqb4)_AghO_Q)V|1U>&ASrL=$rGrG*NLZg
z4j;bb)5zRWZlKP2X;jxpKT_?^LmfSh%q;H;+m(|~%HK1tKZm@1e=d<%&~`)liw<qB
z&YjBi!}8V+?;1T>?XOqaUrX2`%ay<6AFlt{|7m-?iKj6C;hzVG>-6$(s9(tbpO=K&
zWG{J_?E9iM3X~!DW0-csLo&B*P#7QwZSlQt(oah^uVx33!%^@<taM+O-Pd{ddC`3;
z_ec6|G%AhnzTo<?^kPgv(zxy_b-^Th+>1Rm%s6BERMcnHP3n`lS9={fUD!;_MhQON
zzcJTw{@eO=$wL3P{+#*rg;<QGsBSm@4=L@3T5X5AA?kMRhRihM|6cq=DAzt)=DZbH
zg*8}<_1K8E5A;8w<0AX_tTxR9Z5!kdYX6~Ox%S6+?GH51(f$x;7P*A>2by=<<3K!H
zQHnC;q%qFXub_8cQ}!O@f2Z$3&n@E@(0#-Bg?q*|2s?m7ID%tHB8C1R6om}g`_JlM
zvhUaYB`>NkqCKO(E6-mQ_v!8MTcP}8f8^=O8QMRd?+JSJiKO%qoT8sbT)Qx?VNme@
zZ!*?EShMo|9C-n8UCB%26=dm!b%A_)R~=tR-m~dJVIC@3Esxa1e1clkWqtqhYhit?
zH2>GHpA9!%cN=$c9}h8rL0dh4eQ)jP(C^lXr|o-s?2BtSFaKYb|H<ap)z4c#vJdF3
zD~+2M$50GM<vnd_vTB<CH;kr_MKR(SgxXQ=Pn#p%_)+~&J{fman2+wfihpG{e^VLp
zsb|Av$7_7sQ_1O=iP<Q@T+G9NRQ}C(&O$84QY^y?til?!ZIb`DdCp{~vZ6~Ki*pe3
zAISg5<^Sv-lz&Uq&ECh8@mF!5g>q{X#I^j_im&{~&xZBnMr_7bl%fn3h`!D3=>M!a
z%j7O$d$14rpFJCPw%9N1Q}#R4_U#L!LoYwnA;0x~!~ACZ3m%~#Lq+&xc#Dsr`S7qX
z$#D`XWRPS3WwXOV6TR~RJMfM@-sm0Z8N;^LuiWjNF64z(UbKcEc?!wB!@_Cu460n0
z5>IWnJ>DE&K!#0Q&$g{|UK-W(nw9Kd)S?bg_qU^8a?KT7#dX}oZQR9uv@Njyz#`-P
z(TT1N#up%u9yA>PgZ1|uXOTlInr}Q89*QTwJrM@T1IYC{NBE%sNBRe|;W3o{40`B!
zbn{zup|Jn%%ihPg>|>xWvf^Kjum4AVKh7PA(J23pK3cLEy}#7oPxk$V=T26>{#=+$
zPQ`T0L~<qngYen(+MS*aO6YS@PmgOe#4#9km?x}xnm&JWA#%RC#pF_CwGHE(*tiGG
zGRG^Br}w0Nr;WZ-eE;$!>%k8TtNgYGasK>Tay{bs_`>{o^~pxZaSVAJ`xD0i)hR2|
z`1_;Jh9g^shQqH94ey=(WcW$r%y4Ma%y97Rm%@*Gr-vV{n;s6Vp02O&rSR^`lJLU~
zC1Kx;{~Gpgm>qtwWp>!UUwL>~d3aR+zyIj}bM58tJ{L;qeb3wX&a*6|_x{rN`&Y_K
z`RSPFef0A1@Y8m_oEkm?znA~4DCCaP|J8W>e<@O~Tf;+m_XcH>vZjZflBe$xmQ!{`
zUqOemsO+-&hl&4cZUme4XUhB$FNL?pyk!32e+@gA&knn^oBnR%e-8UxcL00ZXN2#M
z`Y+*~8UH!#-XZ+xe+fUlI3(nMJ|-NB=QIwpw&ru;-7Wtq>|gnx!lA*>g`a4TAMx8U
zB*8rq-qTJ>(R=@bKZ%TU`<uudI*`W+G`{>N`mp{aG}ru(L{s8diEQH6iIx%nBhmlP
z@NmlSH7E6<C7$PF)qlD-_PzXZs5|?|e5`*GGK2p(9KB<Vg7Ydr_(V8%oL?gGnehJV
zPlSIss{i!CpBjH=eExF&{Uz+Xb?mz>{QAhy>sK1<=h`zkhYKj!+l}5AE;(*{dvIt+
z$6@=w=_lxFH#ZD<Hf#?Xt_=>2x6HpIn;+T#jh<y&=FZ#yt=s-@^ef`IigI%duan6K
z+8=iYhnw_T_m>)PU8T7GFJE~5822VDjk;-r!)@m@uVjOh_mLZKO<ZySS#@X!-$Cc@
zPlQ2l$iK+bdp7V>@FR57^X|0=Pwu;e-ibD}<H<gCE!*7pCh6EB9cUOOUBWWUJ*W8G
zUU5(KRMpT>OICLe4K?YZVW|Hcj*%FRu_(p_OhRFQ1?`^H6z!i^wSN|9|9C%{M)p4@
zJ7+4UV<u*!1amPDZA;A0Fh-(-?A)MDqV3RqNBx5yG^E-8yR|>GbD9qu|47d|FQ@I$
zsvS{TRrGt~ALolRd95f^oh%9q>9wy))9d_7^kt~eO0%#u>hv2`eAD=fPui!>Z_PXT
z{>iw9c-%XD4Sg-v<NvFDP`5-mvRr#(rE&$;>h~I9h4FZcv_I;sqwD(3*owIReJNRn
z3UtZGF^|W*9ryhDLw{h)UhNG$8E@1;Z$uNCu}d6#unz}t2yxxOBV?aCA^)xC!!ddi
z{r{vbMrP2294g;dhD>?fri*<89m0~k&0j#CUOQM?aDsjs_4Ev$`2Q+C>-qhBTsY&m
zbGU#@xPpH3ey@_f)*HM|_I>vg;U;+-cX1!h%9e-Z00wRIjU(5njnS^pi5|zK#6IZZ
z^pSYF{?9Aw08}orU%YEa`)w?Wk=&sFTR;B<dhPz=kb1i~OrlRl{S1A(Nbf8Tb=bb{
zqxP({_y2ZGnChJAn2Fh__?dj`9hT7NVjg;}=`f${`-T4FwEpi8{l`W9-)F^nr<iZO
zIArD&hx!@Tmj87K3!S?dOR)?qunKF?c2@uYW&QuR^#5Pe|KF?sA9?hk!F>J3Mf&p5
zjFvn4|B*v0s`cmB?DqfCQXAJaAk&UB?fySo9M<}e_1K8b*osn=p#p8LX;16tuPWBR
z>6vQVb(4klU9<@s&ia2ep&2dW%yt_$K(@NBd04SJ#y5z4IDnk6EDq5-M~NFp=*Q6G
zz2r9(^RX6(E|huxm9KgaOS}g@_#{35p8iDVR?$<7qziAop>6Ut^NMCjmuFiS*Dye}
zyjP<>Pp#Iz-lkrEJ+`lXqo_wYUuQ;KO~|1Gd7Qv0{9%6UIxj!+$K}bM54Hc#YX71M
z&G<wAd-E;zlX%V`zr84&BXiT(|Gk641$yTw_CGGsuORyD^5~X_x^PukC42okc@xP^
z)&e5$qN?iO%8~nyAL2>BLfy;KiRz2eKi=~~E$Wa)2K5+l&7e~G3bFk&lpKzc7>l;u
z+W+!k2Vy&`tK0k6F87Q=57}^y{eO%7|A75}hy8z1-b7CRY|Zlhf5ti!;_3VR3t<Ae
zqbj)Z5GK*f$U^^zy?!P;o{H(1iT-aEhuLHa=3?i+{w6VxoR7Z0)+a#5a%?f#&u6fd
zT!y9cdiih7SER2(T>oY?xdv;o9<dJ13g1ZYtWmcdWh2wKqDLK)M|YLFM%~ie@`+F?
zyzhJJ5puh9?_k@OjdJgf3;F*W%31mz?85={KU)+Ik(J^<LLNgBsVMzlmGBHbu1T13
zKTY%;>c<I3dLsYB6!#`9jyuegC*nEVJ`qllr;*j)(82zX<NnS#K8Lt>P>;H{Fb<H9
zp?VJg`wsu1eyrUu&USI*$@squuDOIOxQgp2tbeWjv%THgGoC?vxBJMdfB#qc|FZhm
zvubi)vvjo}&I`!#DYW{(%3;=Y6X$K*MO=64$@ssRk84*xw*M<WsT}1mh-(tYJq*(7
z@3<E7L)SF>jt0m<rXfYU@)>d{;y9m<4Zh1Q%74crk)NhrFvoYg$aji=*Z-|7Nbi;Z
zQIFBC8H-{}z$8q@R7}V3uYawry-D8GwvBs)_3#1yN9$kD6vu3opz@KqFl5yY&tROr
zR_ODw5UE+pC*{JEb%Qfo+@HL@*l$a*4CTfS6#P#^e1Eg}Vuh{3TC7L@m(PZc<YrV@
ztM9F6^*<h$|36etoD{#ZpzZ_dcOQlEMV`}E=a!-@UV~lOgMG+<@!8OJQvPRSJstmj
z$8$wt{P#iSzi*;(f&5Q4s~cL#tmC*wU+W|J-~9dq;_6e^A0m(77?Mb#yjtHo*@PVO
z4QyR9whi*+2_%(Kr^wUj`^yl{ka7IOIkKPc_!9ZnXXE~W&xP%C<j?V*&qU)Fgg1XU
zEUY0f;3}>o_lW&}d|0?i@6>19kr*Cs)9<3kao%w^*~JfVUsxZX#Y3|9x7M;J2W?kg
zVkm}VBt~N_iqZegXTt<?5-JxA3zNyIn2wo9Zqxt#vV9)th5h$-uz%@uQU5mk*KwMR
zYxTwYuX%=k(~s)EoId~Zf97*^Zq^upj<fs%D3qiB9Lr^Xfz|v3Z2#^p*8kI=Uvo<z
zzrOrh)Ul=0$e<qc#j_BLu@uX&0;{kFZS2tYUe5@f=(^_F-0+O_?f0<H8x|RVfF{Io
z0xgf&tj@{ZGX8+9<fB+Cp7q#>%}6$y|IfDFO0N}13Z?Wi)Q@*>v)tnx>E{Qi5Ej>c
z*hTI^?twKb$pgq9_g&V=|H_9$j*p<Q?!$ihKkYlMlK-z+pMj4jwo&6ey@G#jj_-el
z{I#5|?V2P~$l&At-6qFvQ>=gQ+zxrO6J6(xFS*PIPxi=H4KJ$y{oW+bX0pZk*-`9+
zan`>lJH%1`p7Y7d9oorc)!XuaBcDCJ;<H8kDQx9l_xH&CU6Vg=xj!<_BaHogdHt6=
z-+7dX{eGui(@dTr&mpfYD)c|<i-_e?oQoXiBwz5`C0xN(lowgYe4=&)y^2h_&ztny
zxQn=V>C^s^D*uoA{nkM9+d~Xs&<@`{`t)NBC41|XyJWxmZzMSy1^fA${C7+LZ9M0j
zd>mgj4(QALcHbKwiiJ%;{#|uG`DFb6RQhzxME`G%{Wt!9HoaLHUP9hi{?8@nAva7J
zGsJoY^v+&3$r$Yc`eO8)l?QI{kt6!bbMAYR@TFLW6<CEeSc_`)RgL;8rB1A6kJk07
zdv2+NuCY%~vhUS>jj!;pvs;_hwJl_<cVj)=I?niib#J@+wu9`Pqu%9v?;fST9-$sa
z<p}%C`k#&1jIBs6Q2)PbTmZfHDBJio>-^FyP`~%1{{I@bvVMb)_y6zm`yT8=+&?0Z
z1N1{E^gp~~P8Izal1QQY@JH#Zl0Ipvlh(rghV#<*p>#T@2|08ij}thB(`Zwsx5s>j
zPQ*P`yXD8H_IG?fyFaqOS1bRwc{gMnqtQw>PvOHC&pBMcCFB-qBTTgZ0KL<5=uppJ
zrC&$SQE|8XkH%PsYdZ+5)F!x1-bHeh^#{m@D6Bs)N4ka?C+2uinLMn{slTS3>%8<m
z_kG@bx$He1_nwfSRup>Bb;o-`Q`&pY8plC4R(Y?7z1Mr*Gd;WCyLp}80#DX4%Fydk
zhcvdgTdROAT*e<4`*JG&^0$fTFUWsM9&ta_tJS}o)W6&0FS2f({GH|B@77O84tLH-
zjK)|LV*(~&GTIh>)c>e2p%Yyr<nwXjA0^)p(f(kAe0;y3!hS#-{;mIUsyL=2mKih2
z?5#fvv&j<V=4g+Q^Uyg)S(Q+>4N=CCJ>AMIa=zadqWtHN68WF~HnEt#6wA=B9$G=h
z{{K~EpY<!&kTLJ?z;?v?zn5RWPzJoK55Vv1u@PHQvEB3f)uTiyy(}JM7y7jg`}HeT
z&fwE@yblL(2+4c=d=t%kqSv1FY_(~Q#c;$nQ^s-n_~W+DW??C0(1iZ?A0={R2lD9q
z-lN0`viGaMO`IZ6;|$K>0-CQ4BtGu{xFjqmEPHPtafRNw@^^`jivx+P^zwi4PG0|A
zA}_3a)9(^pTYi_Qel(D%nK+P0O&Lhkl67+i5@|ByxSq^^fltu2h4J@%0XONlaToW|
zw&Qn+c61#4U7{0RZ;LzYKN^3R=t=)B(Xf0V(Rl55iKbh>OEmZXF402I(sLJomuN+0
z)$bAy#WR3Gf1~_HQdpHZhSF=tyEpL;r;kMai|$QWdWL(QHINuBtog=3Vk}vV-2Q>Y
z1acCx!aEuV5}oY>iOG(qqOdQFGN5bkK%yJf<D_3&Q_@;1?R8{&k^ewFrn_b)W}^gi
zF%R>x5N#KgKj^rn{OJ`hx{*f@8m39V`)o=KB%0A8&u5hZIkNRe9RK$yu~<C$ujywc
zS0H)zQKCxtDthgsM~T$EM~OA`wW!w)&#33q=N~2Nu>FDjjj{zH?ELO$!+PhG+f%gs
zAB)09dc{}O-(NSDhQ1Z0D2w69^VL_2s|In5T%mox-}t}7@;_Ox+}P#3J=li>C^uI2
z5P1a0(8mAYjyMOf6J31z-N@5>&>&woDg&BM^7p0nKeqD+B1dnnvX6*3QplhQIdq`^
zSK=Uh^`D&}`@XGhL2lP>-k}dSj)^(rxcPzoj>(E|>G%07ed+Yva%~)3qIbTk++eF*
zp<hLhG9{00&#`Nf_J*+DulUZb_1X8Y)(mlc8+UOZ4>5p2<-Sed=riO{3`hUJ=*K5V
zqf(hRmaIB&zcq3KlJrUBWYo&<sXN|VqkMl++a>EgqW7QWYw`E}Wl`w=+amj|7|$Q=
zCw9VA*Tr!w)5)2bjS}>J&AI<#&C~A|sq^haBCIfv`<L3?^f<RZ%RZe??^M5c3^B)#
zz8F2~)jYZr{0G9H&M(}ro>B+bsV~zf)w`&_XWx6*EW-+{!Wyi_dTd17CieF>|3P+s
zsC{u>y`?_OlRb_b)V+=3ZaU2VCJXD|sW)RC+ltC@pU|K4>%?YpZAB@PuNeP7!<+_s
z?ILyDtDguJ^j#>7Kiscwds!VP{KzX05`SOyAo1SbM~TA=9wZJed6f9c@<)ju55AK)
zs4O^8bvN;&m3I>FuDh4mKjU6vU+;~?4|m>A{9wti6ML86NHjK%3k_A{Li5{WLetCU
z)+9a`TJDVrd;Hfv96(u(Jf?3ejx{@`-=T8wpb+<@JKAjy;z@0L=Tx%ej;YtHthxC9
z<ze9;&JGDl{ZciFPn+ZX$xwUp(;@ZYC~cb&p-x{>ed76$*)+m9g#N_eyx5;OB93FI
z=)ISCYsCFT`K+6Xox5))c5S(z_`BXaiMak;%I|v~JW70Dnext@yNTT^?<V%WI*|C`
z;Q@8}v!UYK)}3m6nE26y!QtK49wzqV(8_1RPmX(^u5CgN9e6Lck;v+FTmw)a!-x8x
zF6&bu(~tBq+|#cz&boolIfc_WgLAlmOSpozP5NoJ>3<rc-v(W-=}zc>LeCuiG#m8O
zpb5=rS)?BaIkYYi_DWH>DxT}OiQC9+D+*cRcj=wkBEFrXaG(AVJ^PFJAB)&d+DI4>
z7Wb|jv{T+hauy#UIUH5OQ+J9&Ek-(ydl_cV7KM7$^%jLRdd>MOe^&qh=Y6yM7h|2*
z&z>oa|Np|UQ0%x*`(Xmv`&(mr<<&{?od2e8h@6V)h<gvtB(vmfvIM!TahBvfbn5@?
zcvaa*cD<%-B=e3dU$IuFG>mr6d@MwAn>9RzFQ(VN?Y-}KHY}wtL%rjS<1|^fm%nAy
zb79B0=R(<ded%n|70!7wzJ3*b4c4OLIQtk~=tdqrs7~wuXA7s;!nJJUIx@{pjy7&R
z*1L8iHe)NE&cEokCXVB_5B2|_XXm4nE#JlF@7}Nfe=j>94X?7l(exVo8!hxKa?WeL
z_gtu4=ibE=*T&mL?m<%DdzJ8g^xEBg11PjVzOC&{&mfID9B@vv`#(e;K^*^oj7%ci
z%P)|X?(W!!uZ}@p-5=-WJstl);-mQw<E3AE>!wKm66s$q{d@-H)z+<cT@D@bdYr&1
zoJN~^znwkbfx`NZS@r)Wc`N=8*)T);`3jnbNIzQC{c&%I99qw-|7(VYGvYai3%G>B
z`2VV5;R?Mr`UZ>-xJti{`tj-wb=TAJ|HIUWe7&(gyy^GbxQn=Ve8n5`%CC#UL&pOc
zR3VRGD28Js3iC4DV__Z2iQbF!w;r}nBAIpE(e1r@&TUoRGd(}WdmZIHd&k`qp9`bK
zF&4#`fJvB)shEybmFMamRG;*mFMF<sJy*}SPIw$cQRr_RrEWy+0(IkS(uNGZ{uTWn
z)+L@Pp4litd0SDKOU}c5v_0bgc-iwuC%W+a<FENY*bGe{@_(G<|G2DeK;|5`lD+@P
z{{NJ*|JLy9ebzev;#`cSScVl?h5YA=!dg-jIQ0KHej#r-E?dq2r9LnCzt*wm4;!l?
zd^589wLRYEmqA6fcGxeykL7F^>56-|#W9qX%FFxS%^K%#MJbZ6>3>%~meFgKkttNr
zccFfVviYDgdatsY+#@XZA?za$AV)t$9zpCw=uoG}b$O3DP9pEP$8lFy8-T37#{Ry=
zzd+VLQvZ`r{R_*5yC#Ds<j{dUPT&-tuK)9h9eYpxe@p$Z{?0E@|C0?9)&En}|77zV
z^*@<)oFfbC|E&A{^?y!_GbyYJXXv#z)c?Jo2<PZ|KGk~18OLd|j(vPVc%1iGm|tLA
ze$M;9B<u>Zd@~&jd>0dqzjJ&Yd3w(j-^n!J2^!M;!zcNNkMjkfg`WLTJa^<zv?b(k
zbf6P|xPRMCao)yV+((6X`H&pIptsbo3p`)`la#dA&hUKY|MWHeTd0ru|NP+ajPr(K
zI7T8lnEfa}jHcJFlsA08W9h}He_h@X_GJB^3Bo2}GNxiWdaYM6lPrvP)E89f4<h@2
z^Guj6yaaQRH;1eI=WJ+tb1(g8+CS|1g^m{^r=D6$F2j@YI^9`*EA9FoGVXD(!f(CN
zz9#$Z^H=d5{psq@HICO}JvL%9wxSees6hYs^#92NHS$7Anqpe(&Pp3Hh~uAkId2d4
z;Q$Wd2#z6%w%5HMbPVx+@O1tky|DgI_M`ECWLzt-WsCQV9KCfV|Nqa~{_ku1xA6nK
z@yug?){MvP@09p6XhQy%*6@!K{onK~e`S0qJ@$Q`B2VKC&LNJ0j(eY6pg)<vQ1KU@
zi~ja{??3v0n0KLWy!7wX|Gif~F>2HLg|qsF&+Gp_tDpF!erj=C!WCS_b=<^l+(p|;
z{qN{NC%Vv$JbKX3t^eD9H0{>cO|~4?w@v08x020cio<>JJj4J7{jK-D#5r?{!!z{G
zm-T;dDGo#F!_nh7|C;{qMf$(Vk-{pM7l+a0SR_a2PbDWHj$ufNr*^P$1n+C_@IPh7
zNtfd~GCfNFw=~yCa|*Q|2)`&C8G8Nk;@?~UXOcK8tZnhucde~LpNju*{hwF<fBJtO
z$n)-Rx_D+{HcF6_uB`OUrFXvV+uC6t0Q!9NIL<qcbNssY^6Ls)jLJ>+aUiS46o+Nx
z3dCnI&of;`uRX4h`vdQtjC%~kxy5mg@fyFa#d>VSW|V)%+WurI%20t_*n@pIgd@m*
z=dn#vnE$fHT$fN}{GEE{yVjWZd;ha9ge2Kp=lPO-|LVDtO~|1Gd7Qv0oJRlG%@-uk
z;R5>JXGfE9{hur3Rb0nS+(z6}=q`C54>5q8u<R1+0>16}jep$k&Hq%p+;Pu~z5`+1
zGqi(e>HnK0|0A_q{@*G8t3T6Z2GK??tp9Y+7(@(reT6aIZxsy=Bk7~j=KZ!S<2%UC
z<=)AwAJzZzcn=Em|EtvhY5D)C`k!8ypX+~Gw;2~Ij$%x}BuvIsOh^CUn*Xi+)cf_~
zFw=40*X<=imS8UCq4E{wC%F)du@p&RRm!zx^jdj5^{Rc<=&MlwqO=K1&yZH>F6^(X
zZb-3(YuBkC_?ip*>(c92o44kgHCT)F_;~-%M#pW}*xxtQ^AFVX=%UB5@o|hm4|}^|
zoch-}O}Es)WXmJ<uReqvTG4#bx&NJiNL*V{ik$P}{&!{c&hg@QPGS7t_muk+oj=Qe
zO!Hr4g>x$X*Di7o;@E+G<N*}+zw=uhLx0He5oESW*GlOmV;_3;E%OhwQ&QrqMcrQE
z$e<p_T$4m9UW+E=(1ASKc1r(l_ejRQ3cKiWZ=k&PP2v6Dk$yCx`K0t8mp0_kisp9Z
zxOh(CG|nJ5!Z~A}3FqjY>g*0&pkG2<(?5@H=X6cfPx7*U0^wJYeD#?^TYS5`SlR3O
zjk2GPu$b1k=V1orUosa>9*OTdh5Rq9fA8A3j>K&;d(r$?@;-9vzl!a~(|?(7?WDFO
zo~*0YGfF<fL+1^k{~O}|syw$#8y`b493wFrV^NIg@18*Rea?S>hYji5Oe3QXuJ_GW
zkCPt<`({xqti1Z!Fv<CoF%|i)VPQHs6SL8^Kwg&D^N90}+HUX%qGP#x7S{5CvH-b@
z{C#K`qdrCxy%{Cq==&FIe3Nngz<Ff!eeFQ(E1&OpAr@m9R-oTATScxx<>fyLYsrdV
z#d!zzAEze|8vnP~xJ7y`dpw2B^sR_}sxhr`%wru&g_WTKyRZlQZ~%q<zpR1K`;PC|
zaoqps2zd-iq|mGlmm!;wLr0YAh3rA~1pUeUzZ3M+c)EV*VPzGn2eXUVg{g7uq*3f5
zHfI_cHgY|dx~A+yHa=T<JDd5G<1?r*xACop_`LOhqvH`f_a6J6E!@3?owkUbwt)T3
zrftMi`*RGt4!L3Myo>t3*<k0ycLA4h1<AAezx{TV9(@8ST&Lee{bl!tG~!x)w}myk
zue;=Z<aQX(PYxh^kY9l9*tyzR{=f5G;%Wc?HumEt{r~LDni>9Mn)IV?iMU_!ABb!1
z4|UCOjKpY+MKLB|5(??R_Xp|U%fEphG)QOTHR<<U;#vv+uK&ON_xC@TEY9Rf>6f;t
z^jdy_xc>iidY*mpwEsWe>nwf1!sD2qtommrJ<c&JAuGP;{g?yNGs|}{O`SB+`$i`{
zu1R`bo}KHwd6<v#BEF!vt@%NJvi{v-de7%)g{5R_r~G)!H~YZ1izn;f^?k!!lb^9&
z)m6)!w*srM25Ye%8?hN%(f|24=iPV)dhefm_E~xUb#p`#^865SlW~ks{WSkUkLv=J
zIky74um}5a0Ef_i%{Pcnbf6nu=s_Nh>beHyOY=l|9@!Vw!NTI$gB%Luf0T2nd(t;7
z_KRvG4VI@6=l>n?AIFeF22IGJ19=qI|C!^N-tug(dB%@CYclV+hisUl{WpVs=bUD;
z#qr12|Jg1)j`@rGpZ42pprXP2g@5Ar6n8)0Kw<wszJXJY<9vwIWVCnA#AE(~xIfc5
z`UPCV735C(kIQUMdS|11tMR_+H&Iys$8q;j_ev&(-A3va^`H8oj!Z95SCIAUjk|uk
zkB6u{Yb^vau1%LzXAXKtp2ElH2M={T93v6?-$#>UQSlen5ms&%(<fjOCZoDW{fm^o
z{9430{b^(p`jPqIr#fdkW@0u<Fc<SMA8k`&zqK(A#xr!HF#pf^iadG_8-K}v-?&^q
zw)2`N>bIVw-<r&!b(An^Sty>xSc+wcK8(2k-wJx?LG|x`>krb`phvx$f1v)oqy8n=
z3ai{?3<J3lNp<RGaw`h^|DE+*FB{9?xD1(Y&$Y^PCezvI|FHi5J#BN}O)XjHyGi>_
zGQNv?a=S4aJ3jP%oYcPlrtj)|{FLH(tIoHgZ>r6Bj`oYvZ=6RQ>(RB>*beE=@0b3!
zrGJ6%Z;9`3x$lpBJpYuR=#KUO(Y$F;*dxAuIDkXQW#yCA=H=2m_dfO^9iu1FGef>o
zPIT|!Ln2ec;vSS4vfQ|r<Q@5po<o&<mg48CJueQ&d1M^d>#M3G(<lA5+rEFOL0pTp
zzS|gOq<7i}F#g8hRf`j@J%!UagLAlm&TZ0;C;I^QT=t9*=iqgrxt;&CF@(mWo<&-k
zmrFaco=*!diQ@{c;yQNBQ11`p%cPgl^L2B=ZTem8{MMZCkQ_k8m%kF;`k8V?dw<aG
zu>HTg$7jf)$O+40IKA^l`ER0rI{IkzjB|hJ9`FAA*7v2E{C;fz2Ib;c?DOoLVobm!
zlz+io`k$JoPM?bYH~veQPR@+_XV?Q|2`au-B7eOU=F;b3KKkD^cAkvo#bPp!4O~i=
z-FV5GOD~1(cU}sW=8Atm@wsqcnwI%(1q%Dm3tvUAHGe#{x+JWjuSNax6783gkT#dA
z&TmInemVU8rZ0#0j(;f}&dv&ls%C_rY%=$2%NN7J4b#Gb@l(Q&c1{cLuKPmRzxoU2
z%YHumuxd*9!J^NFz4XRmpAHR!KOLIQOKMv3X}*!qgqG}Q!g~L;5u33Ur6|L(vtKbz
zD-nLWX_o%4uY{u?$Opn>AIvd%zG_^N^&yMGKRozkNa_o#Njw{>*XjTIP(KFJXFv70
z|LgpSkh%CN>o!abe=}ub*d>lVs5t(G@YaYap?utnVdtn9!>$*ngujdT?Y9H*oN-?a
z-<Q^RE`BlW9ycxQTR9{A@WUD5kl&8Pb5_m@KRWp3@b1wshyB|;kCk)6PX=4(!EZ^V
zkimP(swOgr4&-qH72d-s@-)t%dT$B;jCWuSf!bcr9O)b0!Pyf2Df9amnct5lG^1rp
ziDy_6a_h|R_uksZTOR-&v#bw*t{1HjFv|J>BdiZFtR!sLR^5TJ!PXh{e|dfNy$y51
zC3@dity}PAeb@A>=>N@2;W~K}6+dPNo6p@WZRMXV3b%#b#eL+4%rX8<UZHp1v$s74
z=!5q7{x93x9^KdMb&qF+4MpXL!hc>As*cYI!yS)Al0KRoi`qGJ_;Tii^prWF4)vp~
zm4V#r`n&x;0a^MaviVi(0+Le^_Y&$s*KutFbUe^jz;wUOMD-Nw4;gPzSbr$3T*?}M
z(9KUo)?eh`P!>Gt-<a+E63oS0){3ckuPC%B``XblgP&cQ_Iu+WKhXAYPmO4LSy_RW
zS;|)AMreazzPS3nR1y}Fy>%sFG1>nHzbCm2D-ib|Uq$x5As>?czbXl9$@SQX&8TGS
zY$dBU*$;&*Ly}%W?n3P~Wiis1wXg7`|G%1@S);CtW9)0^sFx<HmzJylm#C+-1LnDQ
z5BA{z4&exnA&G+hze62~g8lopx)J%K>|Zp9vvC;vcQ9M`7W?;}y0!Ns`~R}A8g}ZG
z$Ip2CIOV-KbHd{o`3$}D197Woo9N}n7ZmJ&b$0hG^{=`$?j2q!-V5=&b58Q0^T|A_
zZb(<RbnWIR_zUgfZ>w9>)dl;1zjSU>|GMuQzo%SR82?X?^>ShUU7Bq!o>MrDGdPC}
zxP&WcQ<t}|RK}wdUFgP>b-o%dN<W&UGp4o0{bW6t99j<x8)0mpc&_6nZX-9tIiA;D
zdgnOl!F~Ee^sJWVEz-YD`Y|A^a*4Ls_qD|k*G?Wv4oCD2#IcjL80mO4GIOMVy!Sjy
zI#I1|uJJvjUbOxe>Sp==k)hXPtZRxf0h2Km(~)n}-%PgMVsG!1uXe~+h<oC7vv>22
z-W3{N(*{SA^P08CTOP5;$sAhG%U|l{9f<RP^L!1n#Tn;7l#p{#7+3Ne<4WlBu@H-~
z6y@yC*l)0mz5=VT2Du0R;|{x=-pM}iz<T;d^sIDmTil~INE|D)S=d&TqH>D$PRJ^4
z>ZJT!LEnWvNXe_U_`~%P`hISW@=sa6&ssfkp5FEi(t}u@?sHwm&-ewtJ}exfA3?u+
z?hoeT(vwIbgC^wAfjmy2|NXfB<1lNVTmQjv->0l2OP;|wTtM?Z{Y~T*T*Y<dwE1HD
z<0idxk#CcJn|>Eh?ZKCQtHPh!zjHpae|NBdce8(YvVXI~!vWXa$3qO@<Nm)vKk)t!
zv;WiV-zxS$d%uhA-@Tmu|0?_cW%mDv?BDb3U-oQa|2uJJyV?I_tLrMsn1`Pc*H8>c
zav1w}$nY?dUOR&QORtFYUyQ-3-{RgjxW@$hSNLe>j72e;v(JSIWY#_8+`}aIJIV26
zbQtU3i7tKnPw&q?R=>^$MM|8th;s_k$e<omT{|5!F&ibAi+PxjHqWVjxAc>po<$eE
zTl(_S_1OO>{dc4v%_rIa$E6=Rw4(BXK40-H#!@Upa)k7cF_wZ}JHx$U6@3lr2fH_<
zol`e)`0v$EYyG|+8&R&!mq*1{jZ3p8`c`43C_@EyVGpWz%m4f3f7GhK>XZkuj)-GK
z>mS*x*Et7p2uIKpJ{^vcNu<!WP5wVFkD(KVdiDc(O#bUZ!%p?zEAsyWd6jHgE^m@K
z$E{>s|0g4!Cgji&<#Kf-^7L4ybf`B^&`+VTujfJecd!2<PYa9d|C}MuA<1`efxLv+
z2UhSSp8TV5#qm{Slm+#U>&Upy;7wtDzhGMx>5J$8+x`K2{Ly27fBsYEm-{{c`J!-_
zypM;7Yj6&bgZ6rsztj%0XTdY{q3AOQW;ogV%jd#Kvj5Y@D3D`Oj0u>8$(V}in2Fga
z!CcJ4d_@1>&Z3dV==nw*$29^9>&K`kWBIaJSo1V~ezN=*=Jb)PkbBYdV|%Tk$2Cdf
z8W3yg>rq($OB%b6%l~Bc4dbtndSLt&>gZ`?)D88xE}ctVrw=H^{nIwmH)AVG(WVY=
zzo?GtRsW-Fue#~5I!gT-_xfpgRsDyi<?1^1RLcT&+a2{ET2Z;dI0NxiU>Eiv`MU83
zD+h;t^jLSNaDaXYg?)*{o8F-gBaa9>hGu>8Nis{O$P9AVJ`py%|0a58w>bz$jSVn%
zuZw@7hs>ioq5hxfKgO#kQHS&t^}n#V&T-DQANOx`IL_lllwV|jdq(Xj?2q_?^m`WF
z$kTh!Ag;z~(mzA`he-bz=|_&<ia7tV+&a`1=6}BR=gN2SCex4SKc1o2-g9p^^aao_
zp#HLZLmF`|#wB6Rqx1ogSCO0Josl<@^`9LVwM%f@@m)L}f3(dvu*v&B>HUA;{a^I{
z`FHAk18HP@L$R;uzH1(00E700?e9c8mmG@WXuGBTE01?5M>=<C|I)kZ`EKJ(lno8?
zW+R%M*R1{7^2oPI=FqCFiEB@f6whdkMKO|Fyno-t1bXdZc^H%Elks%?k?%hCUzLq8
zPF48!)#}|L*8HaDe{2jKS%UJ<83RYY^(}4Tcb*UP=<~4<i?I~T5c}o|;}6d2D|5UG
zIrq1QT#L@*(s)nWM#wi8rH#ywNrd%&+lbBBic*xJ0{!~*cagp9ggs=Rb^P{`2XF{S
zQ1I{Y@x=Um%yIr3&xi7FSlgfPw%?fGUOt}Q-x?qA73=x9KINPYDt@d#|2NNv%GW(>
z{j5#Ga_B&uI~mKqJiT^^=Ql_BNk4^p-$L9wFzzuQ*FdY@J1*25w8sC@aiJD<XUF-r
z#)ZuJaiRW$abaeR52tYk=Wqd+a0P8!%zsD6hvvVZ)F*#g-#nQ|&%NT%&^|6S?jC0h
zn=!5n^#3o>x4m5dKiT?facKL{_<{4r4_r2Wfb6<s{J<mQ2W}WYfQBmT&o@dRns-~j
zf4}+r_srjSPAgfhzq<yhDf;jArPoc=e^1Yl_2Rth9<SpjZsRUa`IhgK57B(>g)l%4
z`l0qSh9Y;%e_%MhbB45yH_m`Q8a*4NO}~1#|LpP~Wv}aNJNiP{{+jP^k@@#EFNCqq
zD@GMPwc35TrZA7|*Pgv}$8`<rR=Q8mp?aG8N9twgp)fDsbIG`-9uvef36n7u(=ijX
zQG&J?^*^IymV7?Lw>d?=KC3N_IM?gx`v2Z{Gg_4SS>(nTe}ZQ3c&>QnVLlcjCoGG_
z^v+?vdn~0dLk~TVZggRVuuAWD6}bjU`dV^5s_3!rxfUB8Z$^e*k0<?p1^ar0?;l(J
z-v0}IVq_W0f50d91^@SXK8b(g_Y_v~DfR6?n)5>6hvt>$c#(&21jmrmu8(UBCh48p
z8?i4iMbDt8*Rzq|y6-&p6<5mlP4OCZAdh6b{L?7U(4Uk)r|72<*93@b1E#0RKjax<
z=g|MxFN6!^C0xN(T*uS&jp9>u-22Pt!)>zfZ(j&^$-<ti@9_Em1%JX9io!!-1L%L>
z8vnm34!r??X@7t(*e9T&I1GB%cZ#7Hj*%FR^75h(_ZJ*X&+;$kMkPWqeF7%o>G-jG
z(*EfGrR?rwtgi0G->(4%g~L$c2o>s}7%}3naD)_Z@j57`7%^gs5hJD;F=E7@c*PV`
zjDrkk7=$BKs6!npR2XN@JncP?d(Y$ETTC%xiYca;Vv6ZF#fTB7aEn*G!nv=t5w5m9
zx4-+xJD<;5`}<w{`?S_)eb%FAK<18Xb7gDlGs=||`?sG>VY?DY(wp}hW9;1Nn28&3
zBj(^%+=dJL?-m(<r_H_(8Dx<|%SQGOZ7a03FVp^Bs{NfzBksRj;a;m}n=2r#&*`;G
zw8zu$M%)9aVZQ6ay~6h+?)}qTqo06Gr1)Da&XxOd4bBJr{t)6C)Q^ylq2d<xmIp(4
zg8meq#xq!hb-1wqPq+U;Q@#5~(lcmAg=02h3$|f9c3>CwpvyDwMt+_>*?0I{-72eo
zmE)UT*dJ*l`;U%=@*mRZMC^a4l}-baXv6vbhcvynlnq}6dKS4#b_W^HsSj_6i~SF8
zl5ZpSKfFV}i+uls^y9k1<xS2re)0Uiu}u@)C+d!<{~u=~*yF#OUt2OXyf2*(@FB7f
zj|v}=pWri`M3=gLH+l}M{~uH*Kdo+#Y?b<dCHt?A->R<P7VG-*VF#HKj$?E>9apwU
z8^C<OEn~~+vESiy>3)ecJ@5bj`CQvV&qitU2V%WH?*|A+pZ%<3hg8ZJ7>PJ{w+5r>
zjjpYMUow_H9?il@;RM;_dI$c%#y0WaUtwc0(J|%L2AD)%g==sfreh`sFW1jX-WYZE
zIgYhBzn)`Ue_)RI0ev&Kl5zcwJK}LqYNLKjJHTs2;V$93aWC#ikGfYcx@rnTx492l
zb*~)y=oxjhHu3EV^BBg+M@Z8<cbdnLGLL~C_lAmVMAYq+k1Kt*^nCl)O79mB_>YJ1
z2p+=|cnaB;QQ>K_ZZiKD4QMPm=Tj!8@iF<9ajyMsj(G-aunra2ge};HF8%!7eC-}S
zckc@R^(y|gzV$4b<7>C@Z(Dcsf6<N(r0V&<Tlv3er*D_ed*0y=au?DG$6yb=cNyQ>
zb=A@vkXymG7MEGY*GA0u)yMfovHnM|J?7dS*M=H;gZ6^hhn^PhL6RQ#F*rZZziz4f
zmrg^e<H#oICr(M@jN`}(<IXmCk6Guwfj99s-od;0*7}d^YV32$>~lQw`G2$fuXO)A
z-K+ckyZv|l$9vLwA0OaDq*uBBqsBbYdl$O@`J=-p^v{s1lJ-{rG0**rKPj%d)jAC1
zm#CdFI-DhkY<K^jbwiQ+8fSk*;gLul(y!<mo5aU9lsb98ewsWoMf;!p-b5yZlVr1e
zHrhF3F&+~!30EQeQ*G>I*Jk;jFW!R->uaxKUrN~*vgMfkk2ZO|{e=8~SRO~ZTmDbU
z`w8zY#r|jXMeeqixHMB_T+44JJ?<ZH1KBfK`shPus`6!wG62=b&HaCUSXkfje}o$y
zHwU%iYH%yPafN$B1^b$BKT-xIkyz$l#oy+bfnYxUeUDqq<~!jI;k$4*?#1r<q2bjt
z=85bzrm;vpL>u%V-+n-ubidypz(Xh>HY{ZE2tDrk^%(gCp2C3o%F|^3J?c2*8mvPF
zzS+O9Nq7sk;pbbw7Y1K3|Km@}rr(=iB5vR>qr)z84=OIxuJhusP<8aX;br{ln6(CH
z|4Z0;#yX5A{w4fce%vKrZk>8Xc=5pZ!Y`&=5w@voS6UnJrK$fKemUX0_G0_j@IwD3
zVe^zr!j@^5grA=_)=Jvjv1#_D;Xf)b4dur!3+oFn3l&E%3mZ;c8fyL4fORu25C1;y
z^6*Oiu#gmPLmEBEV#mQ@;aBESy<~ll%A*&D?f7Nt;_!yw-o&=Wqrxw+dus@93s)CB
z8}?M`zw3TByteh(@Sjyb3$^{Fp}uT&sN4Os(75rZp`qZXzS$>3)8e0o=Jij8<jyBU
zYyab+rQmV>hmVG~;>Y<0kLqK1G`u6tckv$H#|QWjAK?>xhLfl(G5$b1Km!`J3p5>3
zuh?h&0h)`oyF2Ea{{NT4XEEfL?o)q!7aQ7*o~hbDv>EhG=Kmv$99opYt!P6#^6ek9
zl;b7*e{BQp#TSK<(#o&@Yp%~|dfYoPrEWBq-kXv(#?vPvcS73g5&89(YKDhN;;I*4
z6s{t#L9II1b>wu^jB#I%Z^TUD8<0G}K1eUWjzd4YQP2NpOByo#e|}=q6zzWtwaK9w
zH#%nyZpCdVzsVZT<XyNMT{Zmw1RsIy&F~4xjBu9B`~Nfe|7hcnwxfd&eSUwM&Yk=U
zY21$o&_25`JVd6{$<yj3kI)~(6X-cDef06eGl+9&bIbS@eyc5G@6Q;sOs`(XhTivV
zsQAOPVZ$rWYQu6bOWl+Iier1@8Xj@4?=_BHhkv&IZ`pcnf~bhcALoORTd)n=@vZV-
zJKH~9|5u&woAtjP(%6MPs73$v>LU-E6L78i$Zyo!t}G1=;s^Cz4PM5+)7y|n589V_
z7Pmhevh+9bCgQlJxEA=^^j`NL*JgQ#{x0I4<XL1K(}(JvrQvDu@8NxXfZ9W)p(a%t
zKBPCE_WvhJ!$<T_&^&HUNFw2wCahhc{9dH|Mtkd;@R?&$%bwH!{9HImueimypJD97
zvFAcB`c6Nme)ya|;O9c_z;oeq$9#$5IEx`K(=igGQMY1Es9&-sG%Q*Z8rMG;n#!IF
z2_(^smbuS`R<xlV9jl&GhdlTHon+UN=R)_i=R(gE|6k(&>6v2xKhFQ3Qg=A>To~(r
z#$zHTp?2~b>-Vn-SJ4}5{5P(lUx(&0_rKNsr`G5TS`(&=n~586Bj(^%+=e@lUvEjf
zRII1mC44u^^%Z4X%;lulC5->^J{oXg{MiKc^bzbd^6du0jIDF*19%9J;4wUbE<RlM
zdShTV8~;Q0?d0E)#uJ1b*|OUBAGA5H{irc7e7_W#M&}`MzMrS0^E95p8l<ayKk~{t
zdd$;3sGx5`j{VKfW`k$2#bor&t0x#k<hO08Ju)P0CwHLcu)KC!p2IHTJxG?SS47)O
zCRREBtzn^7d;_w-(e6mLA&nk<^Z8|k-$1$X$ZwPHAkN`=m#ny5yTCox^`XCy5AY#A
z!YB9)Coy1t!RKWEE!NE>&&If`p9@2(#3?U^lOr)2V^RKiVHi(N#3WpW!ONZt*O1qt
z{lt(kot%jqF$d{mo?pMUL+HIL`G(8YA?+hr@!MhN#$zs)|C9r<uekq~=fZ7{8~EWG
z?T^F4y2J9hJptC%$XE0`;%~3}cF6wU8^=T5i^1|W;eN6~nbYXod4T>9^6R_oRKHXI
zjO)8RBCdLoHJ!;PP`kkUCZ9&lJadQK{|)qKum<Z;flb(gx+;DJ8jiCgXV{IS{0e%0
z{huT38ynP`VBg3%mc4@>ZC8GOp9R*HS!H})skV2rcfR%kwkm@xJ<iW-pQlaK|7^z&
z>_U2hG_Z%BU;kz<KRovTX?)JL#&g*}zsL1&)-92T3;1F4JTH3f2-oVoBx<5w=(>ws
zUraZ~)0<DR^W)ry-}Ce17C8=$8{J>4_>B9jcN}V+(}OJDz?*m*@8Df@&FB9kj;Za%
zh4~-q*s-i_@r*7!pUs|+XPH8p-nrg0^ZR?!c^@C(L&Um4YPNQKdhc@fe}Z*>=$|3C
zg#AZmvHM?XjDWam>3vRqiQ3i11du~^xc^f3<=Dn)=5Yv*L{eJKXqx3di{1aBulxUf
zuZ?J$C0~}PBa_Y3)Xxv_(Z4RgMmv8j;{2fOPmK+rUl?z{QT=?qGGXUAA3a;9ey)Ds
zvPk`ZiTXd;&JXA~t^SWRy|Z7v{7QcDCB}tH>ndD>>o6UI=JU-Y`{jol$N_u#Wxp^+
zfIbJeB97(h;a6l;q;C_>`L8T8=#zf?2xDx---WwzFVaU`1Ma8SdB^q0uW!}uxzO|N
zKjy_YkJG2-n1}EP9>WuO3Qyx1bWPy5PqY5OlyiQ3-{Et9d-ec7ejmSmMPX=N#g9k(
zJbwEUetU`a2WIi(rxu2Ob0YiiVec<x@B7T(@T}L?hp<N4aSg+`X23f7hUcxXV4Z+X
z^eq_t?L}emH`dRZIy7t(-i{sEh1xUc>O*_zaZSU9gT^z^8_-;;K7<6C_G<s<Ti5eZ
z8`kp^`KWPCxqSbBH~%-q|8-6hZAhaBS-gQa(UsBvZG3$X-#?DC@8kPt$gFUVY&pdL
zJ;MJb^ZtJ&-`4-6$xi><&JTTCI`80JyoWe9Acgnoz52m>#*7Xh(8nXUnE%fw&inuT
z*AK;2^MgMkKS3@1Gx8*A=nae9*AjlzYeU25;*#UsSF!u#KgV_c>RR1D8V)#apLo}r
zAd`sxe}4Yg3H`zN()n@B?S>BPZPJJQ%Ka;6^Xos7z4CmY@;GybeXo-LE9L*q?td=(
zi*|G%g*Z>R6YZ{JxO7HhG{zz=F6Fw%)8n2p(f^-FpM;!n78z;x;VN;}p2Icdb*QCJ
zCugE&%-8<^UTs#wHzG-obN`wSvJc3Qe=Ff5%kPcya8s51k0iZW9-iZzTX7riz+Jc-
z_u_tZCHVi#)IV0Je~^8p>K|lQI7hbZm;cdLFE7@}{~392_c{N*vr_qzQEq=-zC0kk
zhmidP|CxLO<xT7rx#44F#i(K7Y5Fr*gLR1Oc2$s@um#%?=MBes#M|k;3zQGoLEnYk
z6y?M0uj?=FdylwUG$4t=`@R#}$TWH|@S^XH?6>ab8|0gK8}Hy<#JxDm+;iNEqx`Z$
zdu(Vk6`#G-y7uJ2?}mmC$$st7ACVQmQ%||i+V=F%a1x*6OPs}!SJZzo5`#^~|9?8n
zejcO4XyG{jZ!9?;6EO*K&D5*NYj7Q=qjs_Uzfjppk82t<;0F4QXr3bf&zArB1WhIK
zf7RFaJ;6?7lz;4af}SK}`}cdUDfZ38wzai;FA8&{aVz57&)di@cDx(;b#~ePJ~ltI
zl8t8XbK3k{l&7s|Q?|CVtrzUyYVH4sebjeI=Ptx?Zxz?G^FQ&elnf7di_4F5Tg8{2
z$L`?~*K@D<UU9Kqyn5O2aKG>acnFUmwt?5+F?zIp4S0h76rxZ6&Gm!p7P<e$j&Xc!
zM{Qc}{*m-tn(?%Ap1~TdLj^Wr3$~%_H2)uQkH6maZ1PUFnam=WQa?K8{?X=`c688F
zN7(=U>_6h1$nDbEfnC^xbUcP{UQ6$F?LBCqCy|>VZQ%_1ux?!Je^mcj>K!4D9cXi0
z+<&;H#(B%#7s|C^_QW*kaW3Hb`^@{ln|&jd?0>g!LVX}V{)cQnpx(b$Kl8)#Oic5^
zq2UekO}veF(6vhbFO|p0UirN5u)KvVJ-3(r&+z}H-?pCrFCTU+k~f#g|753d9RKsK
zbjmMQcPHP+`SCv=(9f@*^&$Nu<f^2-)qjw2{j8SY{(bMC`0X>q@joZY&ygSh;~Q@*
z8WO$~K8xfb<=PSD+_-q3<}2=$-!Kw`O~b=zaxBJUBC;P}9ww1j;W|u5|JsYfOmg5s
z&t07}KKFHKFICQpPpweCFEciuUU6ly{BG|4Y4(8)$@dQ(R))8-4;XwkggK786}RCI
z+=aUl_d6+n{PNKMTm4_casJ=H?e_Ad_g}BhPgYCwA#%NOS&xv9q4t<}-*0RJz0ox_
ztSSgk(Vs?h3EvWlMQi}-j_d!|U*0fV|1+AV>X$}xr~c=y*8O+RGgyOlsEG2kd)9v%
z_bTnG*Z+^6`TFnCw^;xEJpK0z^xu;$C(QppZT|mWbN$iLt^XhCg#P~;;{(R%-$xtT
zwNZ8;rOh(EMF0N6D?-;w;{#S3A23(?=&NL3cCs&Ij*q*^^`yPmE#x+A#|~6{q21wk
z`Zu+C)*DyQaM1qr8T->G>|G$uB%0Cgo_9HZ4{FhXwEIrYi_gIKmvV1ti}A>9*8g9o
z|KBlvuD?fIb*VV=4b&d8p91+dY6?78_uq(jgx^JSpXa*YbME(?9b4xe)R!6ej7Bse
zvC%l?R(qY`J?FfS5AY#A!YB9)C(*^mcC)oTWSk4rM`nbxWX|{9vfJF3gt-PazRzy{
zzq)10^V1)ap9}N3G}aY+pNQkHzZ5=;A-jD0m)b{;EU(j-MvldJOvEJoT;6M+KQvrL
z&+k9WR$oI;i%a1;dhb+a1DiXYJ`=eK^6xbH7k%0gZV*?!*?1jt4&vS!x01IZ?xE43
z4&QiOo1^euNFMt-|DIA$p=Zm7hP%bzi~I2a9>ODd4Bu=&ctZFoJcIHd7$;A*PgA!e
zHyC$NaVH<0o)(wFCVFr5?N%3rE%a^3W#nCCy5;4C{P&+-5w?pT_}vv@2e}LV#$E1-
zagQ2<KsF$W!ROV3$aK`3eA8qWZ{SV5jd$=a-oxOJCWZIO{y$C%ACLo=PYNHBpWri`
zMA>rh&AYBX>OM-fpOc@9FTZqHsHMlTi(d*imMI66H;I+<eU&=kD&+v`CbNI+NW&C%
zqJ&)_^W)#xh&cZLY3F`~3gz>L7wpeTAM$HuCPt!*t;)~;-~2cBPdK-p{gdXm#{aW{
zDWs3He@Fd(!2G;J=I0&M7I5770KR$ZjP`?L>Idl39^XAh{bHK-12VsEKbaNIk)vJ1
z*mymzYdkp-lW-NTLESj_$7VK2yHUDLWI~!r|JmH_KMRd<aNKmv#0|*%-{Nni$31m=
z*vvWfTajDL{x1C5|6XQ(gt+Rt>M-P8i2nE8WH}!pKmLB2=Q_(d`VsFJpL9L34w4^d
zNM2Zf!7*{1e$y(?zs9>sct_;gWAY}}osplf)*g8MckFLvtb?>3Lf1a||A72Y_8yV{
z=^4l7$9L>h|30GqZwmjz_uM|2|A7?J=tO>8oBH4*(s~TJeSB<W*#9`rt$k-<ctZSB
zcp7p1U%F8H){(;SjBtM4UbcM=eI1fz>J7-Rqp?+<5?9S9-b8Le?ObDu$?d4Aieo<?
z3;&KC!n;t>Jw9wW_FemLjSuUq#)nP2$A<sF9>3LMi*onpizkH_s{S=>9y2LyTRAcO
z;^@TC;J3k#uMXQ!3=hBb?p_jC>AmlmIwbtc{UrU~hBW%G{a$$Gh_OXiekc5TvNasl
zYj@5PH|=}jSN6r)ar%l-HT#P2a_bf0mjz?ZVf)umsn6`Ch2z67^qXy)@!hbs^}FH4
zssA(leBu8L1I7=O4>L~xa^q2oysuf_SBZDxzp{8E$}8=QN4|r>JB@p}PrJv1_Qex^
z5AWjxl>g*A;X`szd)Hw3*W>?M*!@2l|M##qA-{cs&u|i-<4a@%U*Q&gzw{xm$`2Tc
z(HM*In22iipFP;E?0r=py4Ue<Vv^sk!Zpwa7|ON=bHmtA{de^{tpQhQj`kG(aS8vJ
zUztQRT1xr1eD$_MzWg-(+q3zu$gkt(_zUX;>@_|fecgQboqVcF<Ky|p?IqU#_n*@-
z6E`5;Dh<EiNbfzy|F6;x@jSo4cc0^*XOTH9trPYIa7^`T`w*6^E14%z%V)n;+-;~i
z$p3Y0W5PQA!gnDlt!6Z(+=uIEE1acY>Ib1^#r5uER=C?S_oAcVTJ1Db!~OIJ@DLus
zV|W5j;c4ud7uV3TPXvAU0`sZ0`}}9Y55sHbBi5AaS1Y(FG|nptb@OM3`lGXzxidm@
z>5R~{#9X$5>(#%sKe*;N$Dt8T6I>UPMXqz2dGgX-hYD=M7Hq?I>_Ats_7~5&$FaR+
zUx{ZU&&NKV+yU(mi@cvD?vu>FpIM%*<IcaI_Oha|OFDbdfF#mY_J6Q9WA>h5p_kr6
zuXvu{+NVz6FW!29nbY=vbnd$OY|p+T>&9v)q}N(|v@NEAnnS)DG>YpH&LWvAQmz$+
zCg&tF_J36Wub;>NSfZZ3NV&0Ey&U=WhmHIV=e&ux@ebnNHSdz|;eB*DrhAz>#tQxi
z`FHyd^FdnChIVuyg)}<xfiynENB9JvA&%QQNe)=oAp7XB@HzcU#IZkT$sxPtFN{Qc
zr7;rZSWLtu#IX&jR`nu!uY2o3e*L}Iv<cF)IKO{!+z+z<`_})w*cyQP`Sar;4k|xz
zjq|U=bj-vJxDj(O_`ER--#33`ocfaR`rrJIa2t6CYG)h&Ab;OQZ`|kE;%@rAXr?ET
zP`*cBudaxFWRvpk#EaM;_A7y8H#>##&bc2C;2}JM$M6K6Le~QJc@g`(kbPdxKJ)1_
z%h=}?>@(T2pZ!PM5x(vr{_g?ye--;)%D$5^?rG^fgEd%(^l8T^$1CW)uCE9A_W$q8
z|FQjl1p8aS|0Uy|o7G$SVRMyPsLdF2?Yu3>um3ZjUBx!x?MN<gU#HXq$izJLk5<=*
zwgawjuj@l<zxs*)kMk3{3bpSQsRNR|#p--yMmS66)B$%$a~JlY77a*7-FMPP)=l$#
zXL-Je{qRi({V$SeMr{92JFW*=yn*x#*D%|dK6>vs_m8*f?;y8AT|&Jhv&#LiHZMS2
zZ2y0cd>^&qKOjFuzWslp|HVhbpCCEe|4wxsC9cE&{9XJ1LiK7S(TvZWa}wE4t@T5m
z#gIL|e~iRGZU3Jk|DkP`{HMPDx9$IH<=Zm8NnHOTjsYAk&D0_JlkYN?9>@QTC(qB@
z?c)o?zQ-(b*x<S<9=<8Gsh3Q2+$0QKJS<#A_J2A&Ttf~9bv$x9X5t3ih&i|wx8V-l
zg@IT36J&p#{#kPHFXl!U`?e7Mqx;1@fQRrHvUiq*C&=pEuIZMWLfkX;DdDFP+ak&z
zH!gu5$5zB~_=!{UojOS~8TS~faL$H~lCZ`x>rjDB*n+_y`EJSn-)kcx2S!=@oZN*y
zs70Lr*FYxGhIC|}I^I#=EIq&8%T(Vh{S91Le_(=dc}i^mHNSk;9M?8SJ#|iK5>Bk2
z6Ow4|Hs9DeZ{lsdgKy6Ndsn#2G2Q5ye>4C8X8!-pA%pCKn?r8y&7mb>etOFM^j7oJ
zH=CbcWqvx+8_iEgwcq0Wm-nRgK0ZKg;T&yObHa!8#+9yZ`J8k4pFeeB{ei`<b=jQo
zkz?Ao-W)z5KSSDopCms=s{3Z`Qm(7?=J2KPS!7*PysrFpE^>X9u5YL7t9O0=vtzgG
zLmHjvI_mz8yFaq`jQR8Q%#k_ltNHVu;gD*10V6RQWAV8>GM=1>x+R|P<ZnFRah|Vq
zlBYahX;n*SlH;zzHMkD7#qQrR)9H<~ojYSrm`T3@&4tcIVvPHDTwGf-);VwV+qwzb
zlE_<;pZ|A>ai?+qUoZ|;++DaE_u_s$fWMpnr_P^1ew_|>;vvU8g2(U#V*URq@@YJS
zu449=9sO_T|83O%hqkHmKRVc^6w>rg?E<m=Z;f=;p#qzbUd+GHwzq}eyMX<~Hu`qt
z&d6uTpbtC5?ZO_^qJNa<MGn;69Fk<2didHI>e4^HIiHW)#AUx27AmyWZ@AN%qoeo%
zzg1U{=bHbQCEviCcpLAa{QK%0<a>A@asJ;2<cEmse|<!Lg3oXg=hy%GoPK`&uP^Cm
z(X0$e#&Q6${e8%5z8{RlXpF^pOvK=|z74Yfra9p%a^QyrVeM39(KzL$_kNwYIR7t>
zp@?G#rVG!+`T2i0(9h5RyOBNz|9|KIrCswi&db-ojpezOejDz<UHFId|F*JG$l-rw
z{@>lwxfl220n{!(H~;S;`uX{PkI)~(-_HMgLR`DHgQv)+kv?%zc!peq6u+*=Z*hN{
zb;1?M^7(V<+ozsFb}iR_fu4=pFVM%9W)R2N<>xPsQ9meFKVXB~9oNC;rjVw0E@eOG
zv+s+(QGbf{3N|>olAXnE)xPa-*pE%FVGFj!Yk89IP{~K7=g0ruZoMgbZA$;edgCAH
zjjpA^F}3stG@p?+5@;%;i;MIB+Q@XI#J!MNd~^Qa8^UkK_)_(jWy*dsKmRZD|2_Zj
zZRfm$ckv!#{(qnR03YJQ{69Ybx90z~8lQ*%mHB@kN#_%MhLfmWeQy5W=k)XQZ@#3T
z#ox~VD_bFdV@QqXT&UgSC!@kh`e+Qswe{5x`Y$)fP<Y_0i^6zvA}apC|F*W?KO6ry
z$+1`A8eE6zn28&3Bfd5MZ@=ey*mFgep4-d+N-6&`%0JJi9e+FjV~#Xp+rh2mZOC3e
zD%?R1T3>JQ)vx{k9~mF;q&9$;&5iNf-MAO`BaYvCfP4s#;4z#Z|Mvv_{P@48=+EHq
z$Nv?D;o_gh8mvQp{day^1^xW`@A>vW{Xfm)_@+qk>zc6E8vA9|*;}{U`u{bf!zSmX
zcJnXIwckSDhVAH?XMBM1$bDqSIssWSXIyn1zqFoz+OU-WTgv}kChgRBLNfE6&`j=d
z?k?;>EgFzS8`9`9&bfP)@dw5o^pbtX9c0ExA33xf_>T5PeY!_nAK4+Ca$I_)_5V?Q
z%(>FZ;tjlsT0U~kaciy7&#yo54*gv;Ynw>&)6cIz@SeDD=I{4~Kfs5G^Y7!ln~&%l
z{-o`Wo%w|R8BSum@40S;``@SzS7jX6R&BGsnS^gAd0P7)%3lrPbH{#(vlvp#USK3f
z<Da(w&DH*g|91Srm?2@TbjD*MCLuka{l!)E-u3J+uAyIt96gInrS?IrziDEK<IAS|
zZ6<2<wcSA8h#GpsUVc612;Yh%y&31%*^YIV+r*!*SKdLt3wPsw#P$CkARodbcnnYA
zDLjqAkL{C4=KCLhtWBT3F2=7_w<i0&gH7bXecJG&G}eDxjDNH!Y$xkhYJWfj8tYwG
zH`|OPy&2_Ag<*%|c3}@{(SRh{(6!<l{^11qe}w#BBLB~l{}0Rm>Qb#}b6mS}_rm^f
z^v*^44<0m^!1@evZhx$Qq@@|h{gkP*#J-Cj;n;?sCEtj;y~1MsWAJyz%?Rh$A7KCA
zp|>k*-X&9|#!s(Or>4J;574tv`HVjPLVkP-nLEzM@!OkdXf^&D`EhZ@><XLGTx4w;
zzkP&HP`$`{2xN^m;o8c=aFX7B%-A#XOPs}!|MdOzvytM5(|cRlquth|p^rv>|KP1`
zYz<pm&wl!^`pIk@F3i8D=lxW2taHX=A|~M~T!ZT{9bHGYKWNkMA$zs&_mLUlESb|b
zkzfCR7W>F9#<_MK?khk4-o14e#QFE?2I^B~DRnE%lxFR2>;H?tf!?S-5!e4Of7Muc
z;pW(PpiYsHev|%*IpX5JLizRoFZ2FV_CpeP8&VnfK3}`RBK2?KyO5pjxub8M_dwRU
zu6lKb{5m}g+4PlcJCfb<KVtcDxAX4B{dfQm;SoHBu6o}e;uzCj^sSaIvh*DC^M4P?
z|7frG{Yf+K4WFiW?pFUesC{2LPvL1igW419@9Dy@h91|vX^@ZC(JSz+@nZ$n#}~H+
z+prz^@0LHZgC4)*UF05Quhy4GCeen9KNwe_{+*`xAd5Kve&9;`ZPEMRDhO|qgV!6!
zPQHV8@gCmC2lx=N{p}<26MTk~IN$#EIlXrw-(<eF3;J2aG2e0Do6KDP16eoiziEGD
zvzz6o?4!kDNS!hlBN5vG^5d22&D9X@(g)bR*z;fN`J-=)=Z`Gn8UgwBzYod(N7#R|
z<AAbtrRR@M^xyaO_`eoqfboB0r8yB5mw7(-6o*Olt8fjjL-vcy!*p`+FT=y&pM1ZS
z#`y@}fEzIf=jSiqN^g|s;~ebU=y#y`6x$*k_dbtvu<KTsf4*u`XwZk<sBgQeQoppm
z|0KOxKlga&+=aVwFYd<!cnFW6Yoq>u^i-MuuWzPL-+l(!QvLs{^#AMcZatv?oNPa$
z|DQ|=r^!yowV#?49+S=!cnVJ=UF;Y<L+>3SZT;wL=<ARZ&I)J9KK(y&|KIA?Nn!0G
z?`4_xzr~Y6x%D4%ztjIlZ<y!47JNPb{$=+r+$`L*$bFKVq`3v<S6i=%+>RZn^{rHp
zbt%thy>vE9hivj}5}r%)jC5Rki}=<`_lNcx_m^?)-*~_I?_;6w192^;zb${;r?{Vs
z^+EZsJ@J2$o-B=7{*&Ij()};@fAluQF@+cQKUnHHimN7j$Si7?O$u+2Z=%Mz`E?Pb
z^|tUk$nSq}P(8!{#{Cca@0l3h6+iHkk>Ne^eSClq@ew{j)*d*Yktgx<8P|m`$-&?J
z-{CCzi>1@Uw&H&aTMu6sUYvhj_~l{!Wvj-9$|C(`d#8tA73)u%J3ds+A0J*W`fm8O
zYuYvDdts-3yH{4LJM6YM<xc%kl~;sUH=5)AyfrZ|c3qF!8}lo3;Qwfh%(KRXY&;t}
z3cd>M1z&pBUxwC#FN_`eJT&)z9+Cxr4T&iyL(}n-p|RjEp+TLl{*bYqM@H+5vA@;+
zk?M0J^~qcus{8p>C+wYe)YzYT-&Dc>2>*`Z^pU8T@o!<n#(xXt8~;aGKW}>2<X--x
z@L#n*#rcEwyBYJ{@N>M-s{e7x_^_=+|Kr^6h4QxwLwUF;Y+tC)5HBtKUZ|WsG3+Rv
zWFDM$O=*usnX+Q-qJr?sV(p5)hwNK?67dr;30L77T!+EmPYTn?{;TZ~K@L1>d=Ysg
z=HOQJE>}NWEPvDQK(1Jsh0FxDex)+!xHX-{-;H~5KhmxzCH?_=b&>slyvK*=kKqy2
zvhOwGpP)Bx)Q8n=Ttw{ue^&kfj5!*JYs55dEeva0?H9<8UZ?Fnj_-TQIjP0k_h(pp
zhyD!Kpob0ZMISQABFC22v6uDT?B6Q(e?6PJlWk?kliKK;CrH=171)F=*oN)cfnE64
z{DMO17fauNXOTmTK8aSet=89o4*ECeCsg~dJ<_Q~1CmiM<$sS;XP`GGTpQB#9>hHX
zlEQI}WD~OD-oTr98yD;Y+xL#}yLcb@{fG4{eL(*ZAK??!Z8rX=O8x(^`hU!qC)EED
z*O-XwPsjTIXO26G&+#SBVn_p<gsubn{|~7rqW7r&|1<jkE7`c6`bx-_`Re~@Tg?8W
zgC5t0OFORfl=BLQh29avLYMc^9p4K*hdyM`w%K!2*J(+4j^0mPQ>jz^s^f&`hz516
z#zo%aeDf%gJmS6Wi~TF!;~{e!UE^qs#du7_BwU4S(0<T7Vsbj-{&zFU^a<bRX?_I#
z{Q4s|(&ylB*B{A`e`EvKE>-@cHf7$8<L*FB=A7+|>yPB;-#6L2j{eQ{NB-gd!>#K7
zXhvLr?=ETGjeBuF9>7C*1YI-OfAk=>wg27xzk}-kXhXc$jxnA;(&$9LzWv9fGZ5P<
zZuw4lf*$)HpNes>>l-7V!5R$e2l!O`%etsPYzzUp30trY?cL^Hk*Rw1e|dZdeHZp1
zmJz+^bN?CS$ER1R|E)4_!tb?cKr-sBMWJSyeq4HE<+;B9>}%?3!g24kBogcS|M=GY
zzh!)B$E3(KKRip%&;Pqr9gIKS%ir%)HfQ+ZS^jvAtXs_g)ppQ;Ml>OTB%1MtG~UG9
zcn9y|J-m+(&{e?yM-O_7rC%idS<)|w>HGd&ODliBjj!Lnfd9XU|G&ib(L3kz|EHQm
zAf1o!2|h#ZWd6VSlk`UZeuMUm&*@(xjva_=118A0hum3lLmK5pjKtv8MPW2K7UMC%
zPn$^gUuJK2a_u<ZDW5mCRb3@qcToO6EdNiH{}I=eNgzpYMpl`Bo#QIhhc`4?>xw=T
zH{eG6!})(@@;`EDSuFpf?YQz;`S5r1|JdI-(zzA4;SQvIN2ykJgWkK7eZk%Id-1pP
z|JHwT%Gfh&(E05F#JMLAk&mF}l>B#?Uya9vpTOVF|LZ!Y{6f!i<rn%^E5Fp^vg&m?
zvZdAd#ctyl-Ag+<P8+|7wEO9FKXv;2>sRqDN{tgF<DCA)LE|UUjHmqP(|88&)yqfZ
zI#gg2)}FE6-w5wxLI_)gw;}r@d57GEIR9V|x#0_Ch_--QdIOSZL%VxSlRe1d4aBuw
z;u>df(tEYD_N=n58~q*RV%q>R>HvM{fATxwUGW25=BT`G{=i$td<wsh4`TdHzE84W
zefA@={HGU%Psl;v#b@M6)T}nY=YVm4^tF?HUs$Icj%$N8?f<$xiLcj;FCB9h4aa<M
z{q};P4{37kw^+lFthk<?$4L5U)Xi3|qoGJS&$c!dD%a^rvY9<?8Nn`~Z2~(nhFx%M
z3Tgk}xnKMGX?6uY2iX6;U$?u9&$q+3I;X}wcmFdU6EO+tgfws!z4y3taSi=C#5zM3
znWNGYKV4ikyET)%0ky|23OAB-P(yF1KR2)cR^i)_ta9DR?=!IbqHu?}yKpz|#r=2y
z58)A<?_Yb29>0+%$fuC~b3u5TtoW00N&Nn2=xeYp#$gi%TZV-#WbD7)Mh^UBSlCYP
zz%J}TUDen2zsUP2l>dvpFEYRX#4Pszd1Hzk*MKD2kVX%(=-Mm)pa;F^Lk3yokYE3Q
ziu^zM>+yf|IJWh?{dq$gZ=!uOKbws6e%~SAMOxf@<ooD7rW`0^JLK;^<oekVaUb|C
z?lWDpPgotXnx0?J*|Umk<1{NvqAg9-_!iMfk7dG#&ie#|Kd|o*x#2SX_}~93oTPt_
zFL4$_63SML#Aplzb708+KNW=WWKYVtpNzSP^ht<o`{vMB!^bDD@qM>D<|@BkgY;J8
zlF8{vIi`*+tVhFQ*M|JMWUE~BO7?$+dwTRt$iDVpVW#76z>S!LTX7q@PFw!~`F5W9
z^7ImUd#*f=Tq*y59Q$9){*&$d+5f}JR;1~j7`T%EuMQCRHygNQ__xLn-XZ<Fa5wJ9
z1Bh!0Jw!ed^*enZFPsUF(W|S5geS<Fjm964Pop-Y%zFJR<L187w(K4qlUU>)9TWFj
zN}{gxY^X0g>$kI^5l#H81d`ony}PsFUg<o8HCTrVY{C|7Ls#l-=+2zg{_$VhKmIH9
zA%iS(XxV=@w4&|ES@j=2oHSENZ#^42(LUyE*e;zN*o8evk8n)ES#w!@cSX`h13ihH
z>&+rlcvc-%y5g!ANssJ7?E=4(Z=hzL^X9uRlwUhEyeTd@+kK(Qaf!KS&2Lryv%%}o
z?z-M~Ov-hp51kF~&?_EfchMt%_R5!i@@Iz33dj8&-*wD;cpo3&L-fCXCVWH=yzo`{
zg!~LAG1&1{_?-L_XE7ux|4;S&(NN+#XSpu-l348dyS`@EKin}RF&bkr9uqMMSK(Xp
zkJ+6I?SB(o-#FjjKK9=|wvp{e)c?u+{-R{(>Vj~MG_J#RRCk{VGs&8RXF~0vGok!<
zL&IF*tT8yT4Y0A$x8qw*kV)ZYvTVM5K35ye2>$F6<Ig7y4fC8cAGKrT>DP@L{4?8J
z#6}}gz(yDIBgv$5TKbI-LL1^-z7BqQ${3&g`rsw@_9-;Jz<E7nuk-rIjBu9BWkOis
ze->g97ULsj#u9QVmZ9#1`oU@cUGEyu)a@FOq&L_2&$&awa>uQ}O02^9@dvBv=g04r
z(#w#WuS{@GtV8$7KkLP9#Aa;8;0r@TC0T`?7<gRU2HF2(&xWi=_Gg8ml`Maf-=Yqa
zAK!giy+&LGo3Y`Z!q810yv27;_W$H;*h|LwfBVS&IDms_XFm>+hj9e`Nc%?P8h}UX
zy(PX~9HSpcuAgo2U+339%#Z(H$j3$F0Y2>^zU>h{F4?@F?wk`ih0{2Lg66Qc5F_xd
z_5aif^8R-L8;$(>e|-B^{CDgBRUaN6ilj3J<1hhn4Zs?dzv7$Yi#H&)9~b+rnaxgi
ztAnUJG?8n4fAJe!w^;p<A3E7FsZ#Zi<rjr1^om>9XX`@r_^o%U>zl3afGj;XSskfr
zc$n(AX_$dosMR-7qb*}Lz4563cT5R=E}D;fHsTVeJS+77urSOMKXCQ1FrQq2g;<2e
zSc0WkhUHj+l^Fc%u&|1(JFWd=g#Q@h`A?JPY-vyNAM|E&wPQ+AhV?k#PZs@$jlx|=
z`JczVW3u;*{7+_tvt(|u{O>uo>PKlqJ30{eWsZ9ccB1Ye`^O%|yxAyE#k`pylW3kL
zpFKPzZ1q2tsKQR{Mh)uGHAViM=J}Gn@@pTNU!O~!%b{g~{5@Iz7T@mo4l?EVG}-CD
z+SeO{^OiD2T5%3wE14FTssug1uh&N9$i@2WUy<f9*QBh-kbUy@`cFrO8+?=12V95a
zYmWGh+>2Uq`^f!hOeh2Py0_iu%7A7vmH`L+b`Xbf7=v#O4M)g+9K}G#&~Pk<)xVFE
zajaF@D&Jda2q%P3;WT33UUnGUP8PJtABb}RD{d(Y8>}bTd&)cVelj)QQMY#^-)89}
z99M)fDA)cmj+}sE4F0wtOeXukKRiq!W8X;h|EJQYVFqTQT3voNS%NtCZ!S3xHS~s+
z>XoYt!hGQc$orM-LX)~>VygZpW5Mf<9d982ZvDT?(cz4HTj;z+Sd1kw4=|KrExPpM
zck`w5{rijg-}Cw3$nUeq=WaP+{P}6)&-u#j-R1|>pBvks-f8}SDgR#^K>ZQ^{|Np+
znkMl73;F;2_-1mc|67LT$Qqluf?SCWKl4rK2kjc?|BL<qWd1+;v}s+K|H4OX;p4Zu
zzBWF8JKw*9o|?t~pTYmfDrpT`3vo4BiZZOnMr_7bRH6zyu^R(*+DFL#7m7mrUeAx7
zKr2#6?{kj_jmxL^?l(4HI78oyoc4sQxQz4q$T)X@pZNVafP<(l)c!DLbT~wha{?NK
z57VPB+Pw1H;}6y@^e&Wr>y|42_9@$aGbu8CK>N9K^5g$5_3iEFAE58B=YuRg=eYBB
zxqgJYaJTe}<O?LwggEE-sC16uI8NXcP9uB0H7};h`xE3nbQQ>-@?SRQLB#oean5gC
z`#-J$8q0#t+48>l)LdnO@EPgUZZ%%LYG^2E^(~Wy<OmcYu0fbo79^AjO=zUYy?QqL
zF30$N944R`lQ9KTF%2^?3$syzxwz0iD=*Fy&gpxZPcFouKA=Tp`Ab8>VsZ&KsQXk5
zv!+<MEG!jXhW-}|!g6w8Z3rvKl~{#X2CODy8IWF45K8G~SdV=9d|KX@Co{;e51A<l
z8~wH!Tao?EgiuLVVJ8N)`Rpd+`u{cL!1ca0GJ#f<Uq3AGW97#KYFCPL08`@Q9Kc#O
zpqpNC%W&%i@`GJdEC-sF%lAmm(;iSF{~s~_i%s~4{qH-6hXvBeU@ywASN|aQ;{Xn#
zYqRllmHN3I+e_y6zfG~%nXl)+vzM(W*dMebKmOnEX}@=pbsN84e{U;49r<;AwFxBI
z`PkQf$bTHh5%l9Ij^Q{?Aiw?q+kRpFftBn#`=4e1b7+xPtG0wTb%l0yh7SLcLVC6K
z37nF~X`DepTYTn)q5t&}p^!fC`h*be`UrZz_SwNdUYqZ$s#bo!I-xM^-fT^m#p*A9
zAA@n2fb=ppcGd7uOz&lDdpzIC^eM=V^K6h|+vEKGSpWEW)mLGv-)3MI^7RkTW;VU?
zn0M1}d_mkF;1&M#p>uV!#8Gtx&vw^}5#iVF_mv`RKOP?uc9@&<tJy`?g#1gWQig8t
zKM{U;>_m8J-zj}^|EIpX|113B#HsM&nbTqG@zY^l@qg+6`ZARGueqog_j%Yb`}0sf
z;>)o9#9zavy?+V+p>J)T-{xaW|ETcuaihZvvrdN11=3554!<b7I4tnnLTo=aEc|kn
z`FE>FhRPEo!;XW)!>^Pzi~P12WyM2f)}i{pv=f|?p9?OLpT~##&ErEInk%R4&-=bU
zps}H;YHX;knH2V5x3-8Q%8LJ-FwxkPiJ``Raj)&XA}o>SQY^!AtiVdF!fLc1y)Lw&
zHSuqu<>>TKDy|Hvf~&Q~Ul;0>U-h$;DJ8CjY?`L5NhxzO>ObUq$8N-CY{iB0qdSC3
z;jRVz|0VqY724NF@ChcI>wC-6a|Omf9#HljQU)GT7Lpypac#cXKHo`JNu&I&;;@^H
z>-g7@^%%U}_jaGPKjZiS<NgMWV;Ts)gG-DHc(yoX|EmAiaji%%ljbSV;~q=hWCnYY
zZ>L$TJYJ|gCVQ2;J>)*W?Z-6i!~q<{AsogL^rJ3eeSnm8+|f8i|96T1E!H2c@3?uD
z{_i6F-_!Je`>kDHct@qa>z(?g$xh*}l<@%>{nKQx_`db}?iX1f!2JFk*<M^6j{2Wt
zIF1uYJ3i(7Q}o{b&OJ6MoTi^aZnge$eaIQI@1Xwwsmi^@#l9(aP#IWzdQvE8SN}qd
z-y0J8?On$R;kbWTa+d2ZaUar8kahZ;>*)=yt+7?Sa}tN$-vP(%n-q$iHwO9f&yN>`
zar6l&#$<FYQO@pVU(t&`WcDlHPpIRe#l5sjJ3l@^8GK><7yHmD+%U~^omCvBNNXyR
z!p)VJ`ws2HFr`@gjy@WDG)(i`4AhP(4ztMFs4*#}ebJ;)LZ6Fyn2+=X&kqaey=2eq
zNns&<5pw;mjUCFkw!VT%VX?R+Sc+=<KrAC`4i)qNi^FpI3amuKDc32k>457h;s4K4
zSD40sU3Tud$1#xQfluSu)!6W)`q@{*LK%HMHexfjq7qfui8ywBH(7&vB+!Z!2LEX7
z2QrS~%a8+?7lgg!KJ3Q<^se$fFIVS~Uk(cAlr33g&?jHSJa<U^VH`m}(xvjsK7PlO
zh3dcjANLaH<}{K`?kTPnnyhqxDP@#z>Zo(p9zNGk@Ei5-W5OqJ3SIT`KQ6TYEq1MJ
zYZm$TzXRXs|8rc2I#vp4dgpF?4qa=k!w-ym`l<YVneSYhXHd~~t{tHKhZlu{4&O6I
zAZxC15jpr<V<X-&e&W=SFh+PBCZHI#OSH!x9TFzf<DAMk{$~n(D!w)UV4n8BrTkBv
zUq`%&p1}Ea#E-j{Y0jI0S(uFy%*8x(vFY9DK`-*_-?XxE$f1RuYGtR|$o4brKbaCv
zA7=mAvN(QlzH}B~Ar|5M_`${W^Y(KIeJOJ6X<S=3L-x)1di}qIcGG#j!zJ>6&G4|y
zam!I7zF`@^dWCY6Z?)kT`CqtMI6u#xT;aIEXA8qhaursi6lGYCjo6H>s6-V89xV(z
z$$srP?FWq|q1T`u38dX;3a#|sde3IJFufZ&_C6~vgFervuKVlp4-;G`^6T=Ic>c3o
zr+dse=bPh;_X_XBejG%Xx@0%<>&wX37wVVFrQ9Or&lJ}&&GRSYno1q?RI%qj&hz(8
zSFH71{yZccl2)~GABV{r<3MY1gx-&%XgF?s=uu-n;@A%|iRMGru~UaX=J)d7$W!x%
zhU4_umvDkSg*etOhd9<QgP2!)aoTTZP>}MyZdCs-Q}3r&`&LJgMHqwT&E6RjXhPnP
zPYeyEzRPibpMYYl?<fhA$tjqMX_$dHhBn{7r+k_voNXBzW|JF!sC@e4s4$m45A(4A
z3o-bU;b9TE7}<}_&3$QjSVHeN7HKKD49l?s?F$RSN^%ueqZDy$Uuw1Tf_}dJZ9RP>
za%Yqcr<4(e-W4{B+loq*-!dvxkvp*)gYtR}*?+Zfgd8xJtKvoP>#I?rRXByUe7`vE
zZ5{uw`iO5qT#fW={ZEFz7yA&`{%AzgM)iLrU0<{7Jm4La_<cVPAc2E8gu|%YY5sp~
zADFFuV1_<#erbOGzp?tw#;&(a;NPRoZ|&$f$oD;L{Qm*|J-@f>3_n=7r{4T@^kw+I
zHGE+8#vEB!#{WkH8qtKrX8!;BbL0PyxQ2cl#W5Vm37o=dbnWE-C;0z0{C~dvxAq^@
z4$*Ru|Bp7bqa(%t&+z}rPUp4rug^%QAgzqW2;|ov6kkN|RoCso82UKmR=f6<?txF<
zhy3`H1)~*C-UH(LgA*K=y~OxK@eK<-H<VwkEk}HEmgiRDInMPQQ8$}Uhla)Hj%$*3
z0?Do34=UK34do$Baqd)1!wk&AY<zqE$=~?*d$oV~7FvA^ZDjib^^Zlq%_aO(vQxOn
zGwanZ(nWU9`?}r6v+1K}R(Wm(o)KDRc#cTZJBvKWF`gqDs$92Y>yEk)dUCG!hNgwy
zBT8K3T+G9KEWko6!eX>9EDB4=rC5gLNUya1!0Mu~g5Jwk_UJQQNneGW`gL~9xjJ?q
z`?*?N^|+!?N|vE^R#8|_ZbVIqu`BFj;~C?4J{}sjii>?-&Fo_n`<Ni>R<QpGHnPTX
z8S!L7I7v2_8mq^TE<15<{l}~|d@H3<g$-;#g?U9?`{g<0_gOnAPb-5n$d9k7RQ}52
ztwo+c*<S4VlPTde+3C3chrbj0Uoi*Z(m4PBJ7K4^`)?W@c9R2t)=twjEY#5JF*wTF
zP-JV=A2k2J!#kjNqx?t1Li<wBg`OhQ*h}As{pi`Oe!AZE?R0%)RyeLDS)C{hoBe(O
z2XP3s;%ac19_P+C%pD$%(EHIWoD@!wO$&#IqvDR?I8NY#e>6LUQ^Kcl2Kn)CS8C(w
zl+Q2%MW|zg>r>h%GGC7`7@=){2LB(;(~P^X8ydzqZX70{7?UvtQ}Iv7ziAt2ox=Z@
zR(lEmpG=M8{}=QBwG~d2M!#{_GsuCz7KT~mY?NRw=3zb-U?CP^F_vKPW6zwgyKb7i
zubvq1eX01kc0g*cdI-Ij?e9@XSWeIKAL5+I3-+JwuTBjQEB#h+t$coGL0CnvouU7s
zK-)jPajEA&W_T#2m!Wx~=fB+ZM_j96y}0&_b|i8$(#rL%WF?{v>XFZTmlT95;hl(m
zKe?mIiFv+*Gse%X^!+10|AqY9{cX!Xy(#Q=UJdGzKr2${M%NPlrh4{&yZ*)@_1`1<
zzsdF${NL5;364+GJIP|-Mn*b&5$m%1$eJ0(vtU2{01l#|SpFR6`ikVwlsZ8x|6DyF
z?|+P7|0aZR$Z?0!kL)Ekg`;G}>o<iBzPn@e<2Zp+IE^#-XY>CGx_qw~fg+5-I7~n>
zzFq#xUxz){eexHgt^d3Ee}7Z{_P?fWNIC;Q_Fb&KDU@G+6M1u(B5o?CVFqSlHcBuT
zaqRy*vaHm1w=#tJ^w=k~kX#ghzcYlzWNfQkLdN+|OUY$ej*9zkR*$$TwC^klM?8lG
z;#OcK(ueHdo+=5e=)I@y-+rPbtfrSDH(`$QcTUJSrVnM}st=ch_2fp>ZY>F$$*rhy
zUW0TRGbN!?xC+TdbCkbxLX+bXs1vR?m%m}@oY06SB#=ZiV*cOhyxlQ9)FXjbq|i0*
zX6-NL`J2n%yTZJF^ZhgC`)A3VaLeACLo3?Qjt-=dMkm&GE8h{HUAOf5*Ukxh$$@K2
z!alM>8}Ei;#-LwZ67~xpz(E|sVYJu#Kk^9raTMv@{zLpRdhfhBp$Et5Cy=9Ok#XOB
zs9t_^I3@lx&Y+;%GhBLes1aXCZ(Q%$U<ADg&Gfi7L!#2N!WePw-pM#}0@9P`gko|s
zQq~RVVY_-y7lbLoQ;}bHQhCwGR%K9kuq4!@!MkicR-%6Iy&NqG$>SxVc?|#lwDma-
z=-c1NU)Bd3=i|+g&W0xS=e}W~i#_h1#zs%k{&h_I*J(BgacocyEy|SE)$;#b`5ztf
zb*hAYn#KQ}%1>QU5@tznHfl>tLJ7IT_gHPsq`CCOW_cdX{cQ4ay86&O@%eo`68z^c
z^lb?b{FRM=eOOpPUx-Cmj3ro#Wmt|CScy2VXcbwfOsRJd4X*#f{wLG?mv9^-v)VDG
zD8qVe#AcMY3=3Pyu7mFXu={s?z3AKO{&%|nO839f{Y$IWy|$sf(ETgNQYYL$I??~L
zo5R2l;+%&$_7At7gR(5H|6M8VxQ192xf?aeUO6n(lY@Vl69#|l8=$wM-E+>*e{}!(
zy$8f~BPG7aGwMybCgHuvy0+X(*I6UZHBJ+^6Z^0q2XGLFa2WaZ^VqNk<ktsIusvmL
z5dSvzg|}4lFL&}U$#%bWuxD}JTbu{iDcmK#yNV5*um2t2TK`|TN&iEFZ<+TeNANlQ
zXFnSFmW>Ar!%=#Tc?PvOMn8@dIE89)?dl<?>1R;TqpU`n%}mW39!Ai6XZvsVH2ZJ}
zMZ)>@bJ^3(1pfDA_IIN>2FmO*tkrfm#xeQ%f9o#_<LHg2)lE+DyDl~k=vj60!|Emn
z)ltZ%{)@r{$F!f))=y4GdcD3{aw<}uOMYG86$N3M@C;;^u?xa|tK>hjj$d9s&GU7g
zjjl7+sS<O~)v=ma4G*)NGaDtCi+Pxj1z3nK_M}^yJ&0pX`^cDXw%GrSV}FWb`-}SL
zQT0#usDn%ir^!yoRrB)~NoO&#x69wwrK{bm{&%=AED_fzk2dTt3`^<D&@7ys%m$N9
zGmM=Pw*o7%3TxTn^W&RW3zsV=%E<iqzTb`t>**V@8Cy}O@2|c}`}=;ry)q`hFZW*d
z9nIRbD;-mXo!E^U)FXjboZEl=>+z4q`}V2dWRR^=&q2#(?eFu>^+Eo_{m19ZkJ9Nz
zZWy13OvkuI>d*AJra{kq^%DC2c#LqiO5WKj50MAN9mF9V#^7tl50L#hih)biJ!1Gi
z?W0!~`7k^m#5SE1F`nJ;f7Sf{SGDo~_@c1>rBUIO_`%!17f#3DuO1T4kONOn3<bTu
zZ;ZfT9h>yAcCA;f2RW+9cRVB%iLa5r$B^StD{cZ=j5r6P!L=tAdv6PsZNhPnr@CeQ
zwpIL3{``gg?^66YetKO0|G592?EER1ifQ;EZo)0-(y!jVg8#jYAIUH58|N7o^CQWe
z{`?m00Ii3#4;;}xKz0bHR`Mg!>Adz+qr;D+b35+DkCC3`82;ov^xiT2Z@&0_^q(Nd
zH_ZxX^qcpQdH=ih8~!)n`9a6!{qNoGix1woXmof~T)FWRN#W)b{8KV9Z*=(eyo<xG
zq7h*yUa9(*@T*m`!j3VsLRHoOX1>~t@XIMP!gf@am4ugyO2RKT-W;}#`(fBxGCRD8
zpD&vd2Aiyv5Qc>fuNH*I{m+y5DF#0-2tOnHjd6XJjOF~=D2-()kMS25h26Ug!t?ZM
zW#1lk;+g})!fRu!Ay_;i)S`ZOu{MGUZ1d&DuxWeWIwmyfGixrm)cYF~T2GJD#yU<r
z>T<sNqakH4w2sxw!wb@U5ij9oyn<H|$LZ~v;D65)g;%j|$~EEN3vUds`K=BW_3F#B
z)Vqpr4C_n&_poW>9Q}VYLzCZHu%-0};pa1M2rrb(44bFT4BMvM5Pos+;!yr0We5AS
zecZo<UwYmzEz$qCNdKSj>sK4ScjtDY53l2uvJid~<JhL|j66pE!~U~XMWMCST$K67
zTA+i?ier@1WannzZ$|r+HiU*n))`!GPKsj_=B^}HSZ8odQFz#YtW)0UD#`n2Zwdbv
zrM~N|_W$;s-o*=r;dkQxfIlK#GfMq;RQMCU*PeJiC+xj(t@(dfTEB0C`y3O;H;f8>
z__JfG5045Tlb@ni|L9-HzoN!@4bo}Mj0#@}e}&{C<r<%<scKY6AoPW`!*DT1;SyYm
z%P`ni5H2VCpB){pBnN(65UwV##rJVN+C7IKkUzvtxCLp?GxjmuPS3CZu~hv)Kg$Mf
z3EyfznQZ^BMb=<FWDegxW9e|Gb81VK|EmkbkLiu4&Hvk781A9phh}ymd63N}oB9jG
zPsFt^z9>9MK8)yBKT1B1=x6tgVE6H)@K2FtPjl!)2EX@AekA^9cou6>j_2_L>h#^`
z+XqtG2adD3&PgCiZ$7Gjzn*`Ow)y;fbS&oIBTer_*J|?*=DCi!u49S+rDqlxA3)}2
z8~;~4JiO?CUc$?G1+|O$-|X<K^u`5jGhU<Dp;_G~i3FOkc7^g>oh|yGO@41d`OEAt
zzpH~@5%kyo#y)}cKD>_K;4S<XbxYX)O8tOntkFKJZk{-#oKiP$#_t^Sd;9@^#GmkI
ze2h=gwV3@!kMTae`+QDE*grChoc4{Dl=dfSww1B}=vaNOZ92W2{YPAX>o3yzE3(%5
z|AI^}^8L&j627AMu9i1f4w0zu6gl5k7U$QmyjWaZe}5Et32Mg-373+Wp~iU)(uwuz
z%Z0B*@-*9nCdVakt+?+aJ4~N;m=vz3{{TP4O<2p8m$|of?x&m|U(w=uzBDx4;<q2+
zcHD^{;~oq=J|x^n_A85jLJnR!Bs@qyj7RY}p2Sb_GxYz3{rv&k%U>S2Y<PH9+**|5
z`KYUdpMSqE2)`JA>njd}FI$5{-&|aOOSL0x_(O51uvY)eem}qd)+_Yh1D+pVrN4$8
zJ&O$b(B}QOFH?_>^|VFm&86ziJM9xh#xW#4n~kwj56<royxusZeaay9u|D<VjQBcf
zHlYO_=t3V}$8V6|*E^%1sX#s=lj`oxn|-@gzTrys-<8GTEyw*9zr*iQySzBmEG-Uy
zpf_&T_m4l)|Ac0G5(zZn&*It-O$r~ApCYZ_{4eBRk<x$Ovs=GEz7YNj*_ufqSFi70
zUwZ~ui+c%P{@U_k+H>OjMqJ-)n7F#h`seX?`x6xCpGON?(S~+(6zcmPVf=x<{mxm&
z9~2pXFwS)pyAHBXICIqagX6{@kkwnS2p9XGifh&HZ!ZX==(QV-!zjBVTtaVjOam^Z
zUxsGqCHw6maLQf+xLjQOv`OJg@@m9!57(04M;r&y;~sl)z3>l^U8P=Ls-C{vb)fE`
z`#bFZ`i(z8(?a79kVG?n=$xBy3x0&#aVLI^d(d@6{%7Cv<I6Tnw@UpV=i7H%W_Uhm
zLpwT7$^U26|H)40ReRU>N#`ec5D%mFp!a{cI6O*kJmvl4ar%?U@2BaTPaO6Bkzap)
zvGPX!XPy3!pE>4PtVMa`)1l#ca>MiT?x(}T3-lN95?)5#diDRxb7T4%YuLq%zP>~1
z{|DJgyyBQw@mf3<O=v*}x*{vJKcWY{=sTpSQU+$poN}WjrT#C?HvNn3=vb)z5ovUy
z{~rDCO;?;7U!yO6;6Z6fxBuc0UMB}%92$N@zJ=f7clbR9@4O;J;`&>E5dI_ngpYB4
z{jE>w=hxpVzgph?6Z?Be{>S<KH@mhL^N;Nhvb$J5Db_Ai?E003hl=I%;?Od}`$O9V
z?++dH{P&$H4xK4=!+PHj8oGTyXrd>OT<!b8+8X&OAwQ9SaXtC{hrSpV{!0G>KgFN%
z6+%uKkBc!1m!NJX`-BEGqDdJQ*UinZqf6%3_dm-2Kdu}jJI*M#$n)#_cOBvXuVjB$
z{bT>X{qXQ`ssFhQm*YyLoge$pucr4FC<7gHE&cn*31=tp-^cLZ$?L^ctBd@A{2^jH
z!A;~Xs5znTlG2WlDGWanz8%Sh`mGVyE=VBmof~a;et)C>f^er}YM1GEExjoGnBG{%
z|94DW|0mnTCJV<ogo&-{BIG@eX|Lypear@YU%q0C?-TbEq;|V@`91pO4+=kwIQAxo
zyq`|i9rS$NUw)i|^1f+>=ev{ruk>H?bNQzQ;Zf&3jwkU`{0z@xExOq7ZuE@sKZVjQ
zmj4Unf8=KI|K;-w{eO=8mi=EUeQ7+87f`J}_aa$yOnVF4{1W|Tyn=6yk7?yQBF>M0
z)o-sMtL!c}#-fhigcfulKVDwnNc4BRg!}M1Ht-i>|9e~$_$}ez;&=Ex(k1dJ{y^_l
z-|iVRH2jhNC*->2?N)i5>?;@={wywz5BiuK&_44i`4{{ZU*Ic**L~Nx7^83rF2&%T
z{D(!_|K#BYb>v3;ZU0BsI$@VN_HtZ_tFcagig{`+x;(>f<e$HH(C4{dc>eSjdHw(G
z|M2{M4=LY6zW(LB{zuh6{=)x{>;L@KcP{<!<9hrMH{lli2)84v@BdD6@NxBzYuWGm
zA>qfu6^|MhU>(7G=(VNVRdF9Z&S`8|I3)ap{vhI-a>*0wm}Jvj_rIS1znPy*=J)@j
zC;0x!6hE2I{sZScj7RY}p2Sb_Gdzo~g#LZU_Bf}P?9*qQDdnS+IsSc1J^x>Po8Q~X
z4#&r}{qy7h$+-XDTIrPIdAxvhiDL>!hZpI+{P~_0{7L%D$Su_WzMTKR)PIpN@5cTA
z^8G;c+SQ}OD~@{=HFMq9RQEMY|GTy8UlW&{=Dx=1e<u@D^uO<P|22+FIF4*`oeAv*
z$s>+CVBZ041{)rhPo3L@7IdHseNnFV{?T(l{@dsMAMyUleE*NU*CM^v&F+7_d!6t7
z|39SNdvKQ7o!I%KTN^x5H;lmskLZR8OkhT2U;?wEBX!disG_&%TBtx3WMLO-p#r<G
z0~x4=9q1NqakGUR++c$ZHdsj3<<$+{UP-TBK>{0?h#5#A3khsc1X-9s1PLUNf%bmR
zi(z`~BvY0B<6FPqInVQ+=Q+=NuD^3z>i<Uvy%X);<Fs(jqGQ&Z;XK)o*3vh_B{GiH
zro=Cf)y8xGuJAi6P51Z?-M&HP>NmqRem8Inw^7Z%YV(`nE<Lk;K)Cnm57FfP8qt6_
zm*X+Nrx?2QR(MW^Gx7?CAs-`<UXv5XkcAjnq%TSiu6i>}B+Hh{3;1-c_IZUo{ZK#C
zeh-sfE5bC)Kru=%8*?xZ^RWPnumsDn0;{kF>ri8EP%Y}%2<wrcH=t4Z5Z5O)tHWE6
zIbU?U`hB#rW1RY(>@2V*!F{jS4?a+4=o4k^gZH!#u)#AnVGCMI*=fl3sq72v|7G;F
zasEAIcNhBvQVr_vo$7)Y?C4M2^P%`J7Jrln>-T+w^cde@llcM9bHiXsPN?wvxc+ZH
zc@T$j6vuHAHAtWtZAc=8p)1n5JU*~V8sU!X|IU)<(T_{0e3Tuokk@bnw@`gg{vXH=
zx9Rnp<$v6z-^c6r8~a~Ec~>(<{e!yFU)Z17)8p8F<5v9yl>1GzvAwbnJmfBy9v+iV
zk-VUON35fJkfQe@t<LO2^KktdoEocrfHrzN;vWB<Xl;KZJQt2N^C8a93;#eqxZs{M
zZ-iWW&qZNl7(E|-^fY>r!U%qqkF+7kF{s}6Mkpl5BkrMFw^zI{k$W;4=}kzW0U7)I
z%3svK+xf5Nzh3;wrekk}BEOl287M{xW@8TKA^Al7wN<*wU)eu+$A4x1qk6y7`_C7~
ztNDTp=y8tZB610qp>LM>P8VM?j#s=I&lu&Kt#G}v!Tf`*FVkRIOoPkJ{o)trKCW>+
zp8Xf+cJ7+@N!Wec`T_1Zw=*+;VB;HMoqL9Eu+Na2um#&thH~tQ^0+ii_D$Rx_LGBa
zz7h_S1DpJRm$SoRdZzv_+a8Y6JM{147}s%n9DC|kN1deCpif?XZ9g{#RhuUb%2#o0
zKaTOoxy5l`>?U<q<8t|I&GwM+yJoZ@i4;!bEV>@a<M*`v$#_OUW}Z8D-$$O~S#j*G
zd5W>Ote0bO?U$7iW7!VLj_bxXr>H*;#We}Op>$r|t34vzepIz9cOEErgnfzo8X9jZ
zcf4z2;AKB$*NW|7_m=I(HnpX~Cn28gdc%FU5X<=6<X!ae@BT>Lz%S0d=$oQEpQXLx
z+R%+R!b9>gp5i&eKjc46{ww-83>fRr<&Jx!4U2M92>IlI_GaaUtT2K;3OhG!4P(ga
zGv=Q0FQnI#b?36ec=|-dJwO^;l>hEYsIzKrZucL29O|+@4)sW&0gY(F74I_H?}{)D
zGw_$z|4sckbPaqQy3vCadXYvSnm_nBw0!h&IJNZS(7O2J&^Ghq&_3y7<?F|xbM(id
zrr{Ig&!2=k)Iaz{pZb%~F#40wIPw$r+fPEV=apbK=3v)D`;#O?m`9(Fq<gx?eiFJT
zd=h%_E9?I@eiB-63axWL32h(vk3RY&bd-J)I%o3pUJHb?2urXG9f!Q*_K(8~dQZE!
zVHJH1`u2+3PH{XSu2mn0b^I#FeH=EBn^1lE<FJL?hN@Y<>m%P4W!&XxJmb4w^qqa1
zhO54hHe?Ii)+uxKTG2*tzi9p5Bm3JRsZZU7Zv4vme$wh5VOC&2R{5_El812=$5ErL
zQHwftU;Q06$`|I4k&S4&8p28Ue0hFx4R->~NH1bHE>oA?&kjlM6i(wT&LeFN=HRA}
zLqB~WxjkGWuizSP;1+J<F7D$YhE9JR9+OY;9C7_`IIGQpVaUg;^}i$NJ@Q$%`aV5T
z8PENC{qGa~bY$}HnVi4Kzw|NgD@2t%UUyOcWmg!_z4I2^H9gK7O0>wo7<j6!;@aS{
zcR~?4^f)I>BX`+ncenMHgWEn1Gq?xJvO+Ogg4r0l{k1TMY~7d@=8^NU0E-a!R%u_U
zA4u<!$K&2$%jhf6H&(uPy_ZZ)$O^0Yt-(5Mz$R?LHVmEp#K-pR=O2f1a_}K{fpTiI
z_Qv)Q_VBB~epEgaZWtEgn%~3RM^Sx5nN*V(j??4Z-?}dO{(yX6t8Ym*y8gQUSHILF
zF-6_5P#v*Y{r8c2({JOr*-7#sDhHfxcqBKR7@Hf8KVp~kn=ZsUy62oa>ZZDiOr!6S
z`fsNCzf}ECwic-WN2}Y%ssG8&;nu#LSQlFLr8D+7`wMoQu$P?YB+%Z%Zg7X)fUUoo
zyA5f2+yf}}gdLy$lV@APkMC@;pXAnXbi%gqqet7q4_9tCez!3kzP=&+U|>V|{?<+5
z(E3f`AU<zc>&&<{;lS24VgI=QJ$yQ0efZX$KL|--r?BgzHDULm&7u6r+OQMfuG$>F
z)3rIAcI_;_yKzJKY{#}xF>7PkhwpjT=e66zdDr^!{k2=e4}71416#tO?OW}AnP>f*
z@g>);;2Lh=Xl)3$$Vz8u{nY<)0=Kz;)-pU)+o!V1xoann7}Gj1THkDRs5RbKvvGK6
z8aE;|-W#cp_HrDs)jGISj|xM}8MY$!qBeQzu5j81#<0bV4fp8}@fc6>dVc1Moba5x
za&%q@=fr!D{-d_)FnaYQ<3BU=LO#8IEgQ<pyfA`33QbGpA$|==Y|IN|_!Xi&H$RLg
zCt_foek?h7|BpfuISn%~RP#nCCQC3Ib5P5^U9*FYronTL2=|0=yVz)|Jg3&4n)=`K
z+&3Q!un0@A4C%-Dq3c6w&^PY!UfpCbcZ%%uZfWCKEy8FPW~(qyk?o$*#;<d$wF~S&
zHCN54xtXhc%nkLElnctExYi&uzfk$G!gE$(4c6hy^BXpBC#M^~M7MTEJWDa9Z_wMW
z-O{UnKsK+{|Hte3PqVa*&@oaQ1iOy=?(+-67GZ6}(8)K9!}*?3?`Ica+Zf<CP;8En
zbqRa;S71L5;xLY)b*lVG#x)ox$r^Of6J#@b?s!M^-t>-$@0LccYn#x9BvLqyvpA2M
z>H5d|k9Ar4$9O$IUH|g+{QpV%xB9WIWZMLNcXXf=$#d)j`q6O>e2+eLitOc1+bf_C
zt+j84e$TmtE4YS^%fh(+X1GC*X9jk^cr)Chr;GI2{5~DSa8JrTvENzw;LVWfhc489
zJM?C_?cTenI_AB*y!TH1c>Aiw{qq`kcyA<JYjE#<_q85)Gdv_8qoeB0@RWRx_6E-$
z@ZJdjmpTgRd-~#6J^P7wpjTFXH4NjQj}aJ!>WlUVz!-Y{dSk#Sq>o2asXZvsusEMB
zmA!1Z?~A%|-gC6~n&7?Ah$c+*o5?7`G|WITN-!HqZMd$h>KOg`9x}z<yHH(#J~RuX
z1*azYfBcVaQ+!*`>F~Tx_K7*dn1}hORNgEgtEvk0A)NDf!<;(y`z8G9?kS6qFlV5_
z8Gwywx@kVkT=mgM>LWIQO6hG`{QU>cy~PT8X8)g{9-gWmo+Le%%3ow7nnv4mV5vU9
zPS<z1j$h7ytAw!z>#zZvum#&thU9Af`}NugWY2bO0<xDoP4?Yn|F6;qXkZ^8TeX+k
z=<UbUGl$p*mh&4fj>6f43hYNmfjo+X^vwRc`pJjsN73hf)ADmK;{LkF`JF@!5{ToD
z&15|Lw~frq3)ioRc7fQBPV!6PG~)b&xG&_uLG%BMVg7&Id*Hll{kViHxP}|Jh1<A`
zA$H-Rr*DN_E40sYm2ZdC_5I4X&H5CsKST%pG5Hj6@9FNP(#8`ukPoE|GM*8e>HjX)
zM?~D~HqoyB?^XYkP26Yw_Bp~olKwFa`51vwNUk-$j#uOSsj=#1qzly3#vGfks{geK
zPLZwdYujpk9&vx3&XvaBT^l2uLX5{mRF~?%Vlq9hiHq~!i|EtP)XV;k1|+iHFu_y$
zTQB{s)_z{-x!3%^8-<S<?%#DwzGsih><=(YyJ3p=GcO@M&ih>#|3mB_$2^1l&&9u3
z7$umEIhcp}Sb#-HZj}GfjUK$-|MjYGEH7r_-y;5K-KYKIIqeJC|Impg!l)F-W#kG}
zYfCNiy;sqz=yjvL*G7GL?sdqt%MNJ6#XkJ<H^K&fo3I7jP=<2sK?U~XAP(awzFdDD
z=Poakp3C&9^4abZ&YEYBuHip;*!Lxy(T2DdElH+u8fS4H{kViHXqD%#k+0_W-=N>Z
zZFI*z`W^Z6h5Y$I{?s<_!(G?z;~^g7DV`&om)=mLOsK`{@`2ufMl_-M1Nr+S`FpOs
zyjY&w>i<W_Sos^tDar?Q&+>ev=$ZYAT#IvgGUGlYm0c5F?ps(z4)ff6R8PqcBgiqR
zKfyl1_SaCW@888fafp2a<;D=w^TvhUtBn8Ow~m($tI++i92idy9nA?7$;l|fG|WIT
zN-!I9Fc0&w0E@5$Lv6ABJ}wMwlRmiP89*z@Rak>{Xw@FyKyJboY(t#e+5RNUA0*$O
zkT$QY4{jHRJ=}c_^eW{*i?pggP&3;2;aKB`Q;Z)Xv%Y_l@x!Uc56Mw}^Xap9jLE+v
zQtyOv?h5R3KGAM#DU)Z6ucI41NTC;L^r3mV@k`;H5?<WvzHNf>OV5dG)H_jm;_a|s
zcn5J9M^Rn%wsnthhvW1(CSG^s?QoJ_gQf@G?T#_dN8a@r__ZE!&H&klj^1}dl1!n!
z-5Kky_24x3S){ib_eTo7sPRs<3q518_^%b-R$-%&-h}gh(~nEIf@`>eTeywnb@BI3
z-QJ}Kse9s&^fB>2B>rQ?AE)@WqRl<+=%9C^^5)Au3|_4tx+~1Mhe4J2-KW>PUWb$N
z#6#}5cR?fK-Uac$ihE~8+fzC}D?D|*gZ`Wh7sOxr5^YXBldL10=l?Q)EKTp5>c5{Z
z4ItNj!w~0Qr1kal=|eT<K9Hl(H+i}~{ySj|eW3X5P)Lr)L`+5zrlInpd1z#nwq-F{
zf@*%V$vKE;>%_Bw8nlZOXp*1e89TMwpf%<o)MvRq+;uWB)^%;_Mtb?WyfDx2=VJjD
z;nQR0^Pvn~%9JEB`-G9bNTCmDv`o+j|Ijy@qz#VtDdynw>m02OjvDV+d)c#5U!Yta
zt&g0iZ=9u%j3u783@fk(@$A5L<OXa)(*0fIeCH{?Gk*Dehs^%#i_O1UZmt#C=KpA)
zsr)Zh{`>yBtSQ`mJ}+z$)=(;hZDbkBqr1ZYTVxNAhi`{H+=GSAJtqhB3HC>~F~C;e
z>LC3vj-s3mpyQlp+{_BcxqB{qw~PAk^cwWFNCOxA_h-B-S$RD#tm2<QGulv{r~Vyh
z?moR<-CL*Mo}!;dJkzdGzr8`-n;_5f`?CG$Ja<1X;R-VQf1gz^&~M-tZsRWQBQw8G
zKfez3NN77W<f-GeADYOA?s<%-c#iPL>IDo#K9W<lKQ6M5ons%<-b>M6@Bej7`|m^Z
z0~ebgxX@fJb^q)Ai6?1)Xa}^aV@C*Q6vm(saSng`fj7c<dJh@r@K2;qM&Apz4fG=A
zz9N2=+s#8DXQ29wJ}y~;s;j=;1@$*Mn>()mdv&gI!hH>0>e_qkbEtb}{vQ(b2E5w;
zzf2o##|!P7Ga<|oM%?pk_a-(c`eXV0zqtPKfW8v`s{Q|mj5*I2)&eX-To<#1T!s}G
zy!Dl^ij4NJHPLN6YF%_ovm3}b|7{bw1>ZLhZX5Y({#zOS)%>?|`X2o9{I?2z`*9GJ
zFSN7Cst4Bp>8l>4AIC}5DGOp5kw7dXV%gAyFV8QiabE(>NH59{Npja|{p*R!1o~;5
z#d-AO65`lHt$bdGdL-oi2IW9z|K(Y+{w)Ys+;<H(a0|C_7x$5zD*wxGug8DotIYl{
z{>wPeBeVbVLiu0bZ=E9l3$uNa{EyDj`u64=J`~QNxrC3&r-*AXo|EAp%g^iH3b|x@
z#am$*nU4_|(iRv+mW`G^#!4Skjs5iNL#)uA<{#T1h4DUP1>?!g`3L3>Or{rM8fKtU
z-?PoLCen*hg6aa}$CyozebRU~{T%u{G_BM(-L8*{MAn;OKEK$9UqCKG2Ym^-4DIx8
zYw&xpf_oLx^gg`WC$MI@{_lFvSgrpX!z3HH8_A}wx565~S%(eSge};HGL$1}jd_<f
z2;GZ4XRUC}J?OO-A#LtK-%9fjE?BpYQ`gxV&_-`Zhx_9Be|v-x*Rgk?B8ICinE9T@
zg<3}oZ=CRilSZH4?03&Wbi39w-1z|wZ-v9$nfX8Go$Erc=2wN|^!ks)4JYX}Xrjk8
z2n~p95NbAxKk87A#4+h_r}T%Wc4hT_HhaHoMjMhy;WW-h_e$vxug{;N_acoxG%NdC
zs-(XY(jVIB?G4i30qGB|&kDkM;q>DYuAn1NJ*jTIM(<h4ZlqqmLBEB*wc5M<dN=F;
z;x@lZbMWty_fh?U`NZU7R4r7mek9+jcb{@UM`m5!RQXuB(SRKff<;~NJUeSJ$0<Pi
zmF?J7tY5nzC*;#dU=+q6?)zOx4q7)qo}7ruC_?K2wtKR@OZ$I&PMAS2MhW7cS3P*O
z4@p{?8vDRCbLIcV-V^n!z1Mc{)uMbsoZl43|7ZLC9L&RfEWjcxL1zD_^}hdV^?!jn
zb*%4?G`%m+_rJ{khu7zSZ1(-P`~GBHe>Jd0o}FRsf07-*JpX0FT!B?sgLT+|E!c)K
z4BfW&-`s$5dgU?w7Gpy@*ummnkmX0s-P&mUzdbwbaXqvDkGSoc=X>_&hS&3d#5;jZ
z|F=p#rLSL4#yRZ`C)DxurkihsO~R<aejLPM9K~^*L}vfr&HDe_+3CsDdj0Ej`qvlr
zugPZNv<Ul@x-Om%)TTa-b!*3R^{;xlb&@(zI0-bP4ILBQGgJG6-m_NyyHZ_CKaIY{
z>R<G(=C@w^gI}fV=gEFl^S?x1LDfd_5`NrQ_Zs&NG>#N6WX3r>r>0i_A9dcZzJ>oq
z@t-dLp$TOdv^DW5Gx;sQPjAsC#clds{FU=>7s`J)g}7JDFVDY?XA<5QPG<f8jjZsH
z-q9kB%zYy~ruQ6@w(ykx9P#YH^mJ)#r?iG#ewCZw2;ri35vrdUBPR0^_YkPNtX{(i
z?onuz?&7`{nR7<ZWQ8&O3NaoNF&Raeh8ZYE31(vs=3zb-Aicr<c%dW=oHb58$5>g#
zS6=Uf64T@m8`#b{u8+{BaR1=7-wC@Gl!o2L*p_h*owRNw%Gudr6}blMumPK}1?43r
zVH;T%b?m{Q@yH4?j(@i<%nAGH2XPoj(Xln!fO5iddhGLePiGUS*Wi`itC#ID7446u
z^8XI`|A_p5K%USZ$ejNv&o-gF##(YD{I(fwNFs&PQD3P1MK_|2Ew#~qgEac^E9?LH
zwd2+Ke`W3RZm;^AJTJUm&DQ1LDhd7cp+#(TMb-r`lBT%__k?hTyoMXNh02A}BH4HC
zx58cWKB~9M{~MLV^!i!iHa$B$rawjI{6A@=VYIkn|I~@$K;Go=y@JW%^Xrqt!JSjX
zp_xVD`{#<n589`P!&%e9k<rt_4-d@@KbkTl99=&n96K^I{J69@{A6ZvSR~AjtM7&O
zbMJ+-^w!Joh391W6X_9s*WU|i*Lz)0p@-gm@4Yb0wa=DM47GQLvq20C^?B^%1?=M1
zwk4d|*7SjKuHF&h#DKXD?ybzq4L_w<9m@?T59NlRbq&jmlO2CDE1bwH4L`kC5-OjR
zh7s-^g)t~Z%b8K3+1OcToNW83(0<jp7{<Fc5uZ+&6uvcOQkYCH!me{u!tNbKp?vI=
zuoK_Tni{_IY-*V1S~0$xH9dT`cSfi<FfHsGH$8lB?#%G{*qLG1UE{@Uvfp1iEBs((
zaX9#7MmR+O;fa#)qq(J_#BXO~$9Cm3j($**nKM>%@BL8w;(cwb_k9EDY~1_$KhCZ%
zcwhhL{V>OG=3zb-U=fyJ8CD?azOK>sKS0kU`yZ6rkKl!G{^Y&T_u#$Iyxsl>AK3o@
zt!P90NA^E(U+2vCL*<T=uu3><unrqgy|W}#9V!W%==Jovu9C2Yz70)#OSDZ&jGdH(
z1j^5|yV)zC%(ZfK*h63sS%G$XcSC9Du?NI{?t@4lD-C@Uv>mESL+>H`KUh;)JI?+C
z6YM`wYX5;r_8*vP{{gaTsc;MA4{hAoHaP6}M{yiAF|4G%zx>*zJ<>ha_#0B%D!oX1
zMtn}Quv%u8hEr(GD-CU))qX<XA94S^O6wgH!pf}gQ?4}As~f~0ZS?v_;xBC_=_xd=
zW_!FY{+Io3pd_5;*E%IToF&gA&VA}9FCm^`)_v7`-84s!`x??)l?zCn^Pc1lezy?M
zP`XW4%`{h!ypL-7L-H}|N6N?I*U+x)7QfgwY{cM0<wN5AFfjW)a~0kXPu=?*;ZNm1
zWWJv^ZCp<?jC-i&y^v3ii2C^VLd{hB|E@Roy87i9Y{Go`2J__`$tGj6CxsRFPD<a(
z38VaG*9f-a!{%7f3z1Ymcf~qdT@&Z(r}EUz1uy5U^{LyMuNs5BY3!A3bzj?7bvruf
zoh#M9v(&%l)Yl#P#s0rt`u}LWuK$1aD`C86OvDWDF_|pFwCF}LlG+em+7jKy6MB#`
z=Fp4R=MOaK|FbKcLhDZb|6|4<v{O2?S2{QH(-tTZ&TP!VJal-ccJDr)-lJ~q_O1))
zixBrTOfMD3h2pBMv4mfx_Q*1F1>)I(tH?E|^4mIXkGQUX9rp$_j??~lqMb28J7cu?
zcj^D*_4!|=%H~P_+o|ela+BYD*}k%cdmGA--e6tf4)cTdv48Aj|2V?_v6uZreUet6
z^eJbWmur70H&3AzZD_w|e$i#~i;%G&R%M6Get45#=8HYTu0T8kES~+<Gr|AAPCXl+
zN$+Drk9&LfqH=)!iT`07#c{;Gdp!H=B)wjKiRb;)&=Y7{FaPjs*les1wcferfVoK*
zL#R7pZqiNt9W=4$H1bOq6@+HLYeNz#oW@z4N8EcgxmDcBZtb*;UF*6$c1|6P7T=~B
ztsnRXABsEKwp8574(@(oT*4Jx!wn2shkuK_jkGbgL1%8=r4MY&4Vm$i;qsGb-skra
zt!uT<$nvwk-#GSHdbs5KjrLB%wF|K8u5$p8zWMUqGV{c*YQMQ}81gX!Lsvo=MUIL3
zeEFM=eW1<wKl{s|b)|*;$73QUqX^Sbd55jyv~lMnSz!iuF{%$_g%WZ$>UT<GXjrQJ
zSSW3MD2<Viz3Uv;=V3k;U=fyJ8EV$rpHDxz4)yx9iFW(C^}eNiu+Lkq{<S`J%M<<U
z7jZ4W{`CX>>qGj}$MmoF+E-2=zH6uc`M%gcmp*;BUZm-L*l|pKh!vg{+t{nfHCTrY
z*cHd7v==wgw_qE}P#%fj{)j$2y#o7j5FN|C0}j)B7JByfx582Sar8~sm;X?o-X3+)
z22r_G-<*F95@<&CG4>C%(d!TBoA;W>Ku@9Rk^cK${r4UE>f~vDXK^0=`11JLCGK)|
zm@DM&yKyhS+;EM41GjJ+HPT3}G*d^`Px9ZC{^k6C*FE>~5RdT`&tcJNSuWUULl?TQ
z%LDTBuk3#z%ob(IDY8}AZS;6HOo!)nK2s+MBOfEsdcfLU^40#dW9Wq#k8U=(IEN(l
zLcK+%5!bR+KgtdhU7w6N=e&rlypSEHku%WPtFGFoty!h+qSv)+XOqRQmEiSyPyhXF
z?zHwq`IwwAk3Js@un0@A3@fk-yW$*sV{2>Z>#zZvum#&N)XX+WMq7C~8P^5xAuF&S
z2XPp2-NRAxI8LGl0}1~F8SU$3)1}eUm+g<C$NK5(?0FgQDUy!j{sFV}5%SITzw3Qg
z<b@kd7_ZJRO}ekgH;Qe56#X>%HY=CVdyXH@#;}BYo*Z0YE)01IS8xqCa0|C_7x(cH
zk1=%ntx&z1ZH4|ERqL}u_$R)9gLysVFf<Ox>xg@PG@$MS-yiv|jX=#2^)Kpb)l*1x
zy|ha=(wi{KJzuVW#&8#6Jkp!AzZ%s4?dnK&gB~))-OJXVMjx6_=>OvsTG58MF0P|W
z{qMPRymzz##QkI2=pChO?ZTdnXak6S{~qib#_p~Elcx6}wb64B`~F4lorW1GM)g+X
z7p|4i>xCU{0JG_H5YN<UM1!#7Ub~s~_v_uWTKxC%^G*#r-9v98=lRWiEWjcx!7{AC
zDkR<0<=$?;dwu>vgEm5y_uQ`k_d@*7iT_3MC);j{KiR?ENwz*PwlAD@*nmyw$aBxg
zH^LTrX8q$V>(c3E_~rGF<@_rBPkYD;RDWPU1F~GYuUaf#+r?|H_fZZV=HIwnytor&
zgX=X%^ncGt7pO-94f1cJeBFc{#_G#()Nen1W{$vq>+k*V$GJ};dCvIyMSX0t=caZ6
zz1J9j`iXYJv0seO8>2skR<t3W{n3F=w0@{<Ae;o6(S{Cw?MTvlb_yFQ`f2ph)95|K
z4`=yR7ML?b_M@79iM)a;dOY)|9@n^UAhWL!cjB@(2x6T&aPs9o`-5?>ec$Pp`vxzC
zaGSh~`xq*=em2U->P+%!)SsGvqAq*YKPeC6!gK!NpUN8}#)V<zuG79(VqD0lkH9F5
zK_SMY=bU+q<|THSlbAFgvG1aJi|D=Za*coMIB5qhXhx@dJG9T+(T2DuR~_Qo{u(rG
zmv+&B1SWdMWE5c<W}p})n2nkT_V+h7UpL+OH!}O;A2Y^m?7C^++acFIb1)C{u>gy(
z1j~@DHU5q6<;K|;8du+FTpj7n#@W{zmw#dW`=ashbH=|LjDL3-|3(MB(>Q-CeT8sV
zVGR!YejVf7gLT|J!-YNT?XZC!_wVo9ZfqTy{r^WAJJ*J&eDU_nd1wpG|6A(1dSR3M
ztLSy^t^erlm*d~c^G0DcAz{2fv;Y5c<Nxd3x7vN~O^^-TjbxK>w+Lq&%Hs2}2Nl?l
zgLu9EcB^(CQrd#O$h7mW%KtmXf1mi1t;fWlZ0GJEJGI4HAG{q73+E_~<0LxxwWEgK
z^P%{E;JxV0=;Kav_mZiF!sAz2Yo7x$g=+d~@+_+8b$Q}dU>*Z^KN`84xD%trldPGb
z4URh0BY_4qq6wG$<_fMMy}~#Ec?)-OAIZJ)Kf2}ho|q@Q<V~dIi#{~pF$S;vI7Md8
z_oKHfCpxam|GmnVTKPv`<<nkm^s1cjP?+&t&W;9a%;@E})Nf?>RN+q&eyQ-uH2OyC
zf7~;V+4Ws()W6z_;filCU>_y^x%B$A;<nPfW_muFmWms{1|&A-gc1BkVGIf}9s?!s
zgo)(fY<oG7MVN*-9zTODMhRwP4nCbE4kNV{##xubJ$O4O%qL?zVgWg%4PU-Wn@Rg1
zv;KZgURc6^8CGBw)?ghrU=y}rs9D?Pq&b2&-VWQi2S>0+ljYch3PhXoe)1p=<0vxg
z57a%!>2XY=8z<>C=zF0KMI0lKXT@jw-|Clm#$EjZWk$QQ<FYd3g0kd{xoCcqKr`Bq
zu_v_nzmVikuGIfVH+uA`Q>*pAjRB;|zH#dRy~-|}(l`H=`G4-IRL`Y^a~g3T+F7!C
z;mi4d=jpHZ-^tYf+oZ|fFU|j>_q*p3;@&@3$hh~<HSz{Jv_Ec<x22Ds54@v(fA3uH
zh(5$UfNHiXJMn6rV5a`}Enck?yzTdQaUT!y7*FvW;i~xWQ~w`Q|D&f$TzX@><V*J7
zXX3A}KXqMQk2c@69k0$ms?4&sOE|-jj}fRIEB?OAD0=;H?^bF|mtKgb0`G<fBru-e
zL`+5zzTE$q#$7HA#(fKR-wk2c{I^0ezY@&G9Ml|B|7(ZVUHnD;Pj8@S_UVz==DBA+
z7GM$5CHY|qxeP0iydeLh`?~z8K21H7KaoBnf1-Krm*(d!SO0Uje<XjCo$hO$%l0pv
zHCTrY==ezek4^NRy~5s+6SmN|p^rPw-AkqpC_ngBW@&?ydr&>x*Z{d7RrI=Z%C(C*
z;UM>6G;%j_CzL-8WM+TwXkV0n%Y!)w?in~K?~sGWUTeq%nlUtu?SM=oh11w|%lv}J
z+VS-B=*J}t^}i9WkOR%qBN@;3y+Pi>ZQR9uJj7!>#dCyzreB9)i2KLw9AkfQVdZlV
z7HV%yQPzy}zWPfe_+|PZ{q~<*V;?}{`vaH$NlwQjeY|JJc3khdF8^Pa25y^!&K=vN
zY4jqVRa5iG{%NQ~JrdLPZPAD(ob*nk{BArZVls*_4Kt9uXMR7r^@V$OnisF%+^cV#
zMqj(W(qi+^mz#f1wz{wFnfd4Fpmz?KKfkpg6bq*WvoQzNE&AVGZ-#mF`Xl<^2h1~|
zFF@QozYz^cc*Y`rt!xfU$YtobXRZOc3hnf6Hj5s%iW%auhF_X(qL0lW#lQE2ed{L}
ze@7jD<@`hMyv}bnU=y|=mhn4=zpSseaVKYqKe{i7KjNB%Uc@v0`p`U8{AY^4_syJ-
zFk1Y_ivMu&M`i7s4iJ`4%hV5SJUiJ4svE=~d-&Hs@^1I6UzlusVGO(2P4T}h{&&Qm
ze}#KmAE*P!gNSoK50gg`_xa3>ORtxnxlbZZ?_2J_k9|;jjXC_a=JnT2kpGcD1L7W^
zO{npk1e(!?pN?cR$8qm{Lccb-m;D_XJK-UHT7CQ8&GP7W`JZgQWBmJp@o%#Ag}Hb1
z_M7Zve%tBZxPQfG=e`+|!cO5ds!y;V@;^(@>_6%~&eQwR#0K6t#XBO-A^7nmdpP<Q
zNAG<#92;%F!MR^I-tyIOc<I-}k%eCm-{1J_;Ridu77i}{^>FCHuZQnle%m;gIj8O4
z3<tCU%WK~WJ14ypc42qR*TN;wxPniY{zmxL!rurLbALVTtNKRxE<Wq}Mkt&0=ULZW
z|Msk}h3_;7*PfAwq@(YzeJlLn=3C*0YeV+)|K5dP3!iWQwQy89H@M}^aEt%IVfzJ=
zas2p)%B~+N4+nD9$9wX^;Eg}ax$D|}Jj9T)XQ;xQfDL%g?x5_6;p~{8oiWR`wg)+J
z?PFZO@YMa8_D@k>cus${ej)td^8Yz$0J-#Gc%A>XP2*hka`!(*zH1{e3S&@+@u<<q
zsJ&x+U_f6333_I~-pj@Z(5(O6f>UTkn>wx?aSeaxnK*|~{fO?Z+6Tz&zc)$UJ5n87
zp#J|bFHH2D$tc1!RDWds8D`MycdGwTOfNx`y08&(@3I7D^P7Wtn2!Zmge6#p_^oyz
zmX|BI<Csx7*3j3X=A8Qfs%Idc^_zGh>~U<PquEUgtj~1M2JEu-dUqjvJbeqcp$xCj
z-`T7EkJJ(Q3Tb*Dn&q<=W$h`l^&b11aygdG9rR9inbzfOAHvy#3Y4c@JLaCv`tjU7
zUH-q8tZ<Ni7=5L}MlVt)%s1i}*AE;gPony|JWVE0HC?=(uq(ds9^7rnocpqsJxDpx
zK-N4{|0A<LM}41|q+Xn=jZDU~H<Nyo!fBkvm)n2mxs!*~|HstPC)EE)ZPd0z8hvQK
zr~b#OC+d22Z`)1vf4lm>Mg5O-e@^HZ&Lv#K4cx+Q4Ba*sMn)UneR8nK{0Q<fGW!V^
zKF{g-$g}a3-*fbh6y{d#gN@$hgO|@Nzx$j4K)s4#=uq#sBcFaU>v>Mqq@U+hU;lZ|
z&$6E8jNo@-v^`SC>06X4V@CeRoQAB2IgL~QeNI!>zt5=|`R6&cNB%sgF6(hl{kZ>_
z)0TBNr+xLm$w{Amo>PwItou1Fi-&SfeKeHQy7AxUj1t}$6k<FkVls*_4VnFaXXLUg
zvhUx0Bh27ej1tU7*-YuTRQe_7anHvB#J<7MY4$JW>LTu4;<$U;i<~9&YGrJkL%EDz
zzutd~74%ie%&T6i{9Gt*E0yp2zR0OP^hHkHu`hD!$wbQ+ISsvE<TTcPk<)}ZezOMa
zumPK}1=~=Dq;J<X_2)U=WKZeOb5dk4cbe?G`171*VYJ}XjxTar(T4WDU*vSG{drF3
zxS!{g3u9pX^PD~8;Fjk(736*##9<u8ahyaAGWD-?^(lt(U*xnd_Rjp9k)Ex6Ms{rX
zjICefB<Vd<zsTuEW`15mnk@JtC!P02PVeY1a#E-~<elAj8fS4H)xAH@sp|T9PCvc=
z!OwH*aEX2eP2$ve?&moTSAU+9z%_oYkG{ycLEb`#?|GZNi*|bVT;;+?(l7TzWX^@-
zPA!&)$r|rgJJa_s_5D8-mk)e@vT>vDkH>!V6weW^OT!q3e2hTyL-S9B)xF$v$doX9
zH@|G-MO$;TvcKh`@`r4_sr(_^x#Jx1&h6%Zx4+04C7dycHrPV4`ofExDvYPst6$>W
z+sX8_@$n|^IQO=JjC**NeJFh{_J1I*|0{A|6<LjG^vwFdQTpIhw8JMUUz9Pg?F;hR
z4EGeH1hX**^DrL^P_te6q+F>(JrZcBRX!b1KB0Ms@@1d$Wv9HZylUGkZ}%!+lrf##
zm0#YYOZm`E_AFN>kiFb#vhNvlfik1DKpyg}C5UzBGI9mt8o+LOq(}Z)#k~e`?O)$=
z?VQcpIrMe>HeeIBU>jopp^S`cfy&7}sKAhVCo^C77wz}`{4(u6bE$WaVH+sQwPszK
z$MJZ7jea{hu<47O8glTV^g%YG4N0VM8fVdZAS;|FU#&0er(ePqbo;M+q=(dc>Eng+
z`H8%6Pyc3$`fss%0vY@FA@$!ebu1b82|ehy*Kh;3a2t1V9}kh4|Cgt4tuF5&Q~Lb9
zWM=%mRR8~p_JKO}6xq7k`1^L_?`#Dfi1YsjN92Xa!Wnp&8=jJ{+t=#s=iK3+`$oT*
ze~?Qbvc_>3xtl$9m-cX_cg^P>fl(NPYISNA3hDJ*#ciX0GJPWAp23aJ%rSUmt%3GR
z%>nU89Wv|xy2QUq{2Rodoa{G6n1&fBMhRwP4w9?oKi@Rd&i0Kn`}%Qb_W4&AH{TS0
z;hYj)E14PpsMS6?@^btm?uj%{IP<Xpi_kGi8)~|_1@t)f(e0jP^cCn^&z8UX<=97R
ztGNdJD#djTxenFt-$2Gah%@sKT#I8MX=!o`|HcX8<v&l9iswvmKc^08(e@|n5c~cK
zvgwI@j%|KZ7Q@3HRA4_2;xJ<0KY2#px-O5AvG1Sa7yJI{NAjk87Ht4=@BO$&FRo9I
zHh@k<8$j8I*5Kh&c{ADq(l_kAO?Kd@@H6K&-WK*=&p=AJy)kThv;|ZuhmO1NBx;aA
zv;oAve>1)QoU#LL^dy?-(FPFr7>zc76u;9ri;2?Pkny|o^nP5zAiK#Ga-bqBTqAGb
z7VhFcGW$PoR{m{gKbUQv?L+o}Ed}ABYmf01&k_EGvKzyYj}aJ!F(|}%OvGdqVH$?|
zwa@SC)7uAV2KV3?dytSNn2kATy<;s1Szi8Dm`^UitMz|N=yC7K%=*9M`C%D%-vnuM
zmNH<vv`W@I(Ernis?$%cClj%6O=kAVBX4{6PmSyExbv2SzTOIpgs}p<4jYfNW;C<^
z-(3CdkMy(26ng1tvTv*T0n7D&pXvYV&$d?S|M%+ul5w4V=b<;lDq#(Y;~H`uHeeIB
zU>nlDe;HYhJ*dE-c@+D}0eenV-my0z{V<N=II8(oJz=|`$2A3Y`r|e91mc?d##t|I
z90~IE`Fjh+AFuZ}*vS5YX1{4e5-EJyzIHnLU)KLeH+nobHK6~GH2TmyQv7iWt!R5B
z{^&p_TD{wO;bi9jCe25o$7|6>a*5t^RoJ*fzlOe<!ba~o&%zCUmB;ii$=j%Ix9=Z$
zA633}w4KB`3=g><qjBTQctzVuoWt<C|F5lEzfayjq6|ReUS$B{`EyVG?m6QA+XlGH
zaxo0~c-{Z^o1O#e#V++CVjH{<&Cm4zaZ0(;s$6NissB%QoYDVxUu(NHw!#^OF(^dG
zSodgKjHmZ_uWn4FPewdbJMDU}>+!7ZB7SiXylG@H;@P()<ZM(eR;FzgFYKD9jun@<
zr&yD+B|$d0{#Tv<HAgt}FdqvL&jKtvq92&a7jf>9@*l6)|J;<fw5?w6|0sSfz2d(^
zT=Dw+MeVk@|KlQIF2OPkU6J3_hb!o-qFetzeV_gRzVj=%2MgJw$k-p+5Z(4*jQbC7
zqHn=Al;PF>!{zi>`w#D-SD^2@cf0HzNBe$=XTHwhzaIy27}e(LRhj>Hl%CoD|508z
zPCtnzby1u{-{77E$`)$>Emi)lWQ)*dsBzD$_4f&SX8qqH<@W>a19fi-ao?WwW&Qt)
z+6V4wLlP;R##s!_&JE|ufoW{~<lw>FaEZKvYq)`1h<)GN<XzmyLsY-8e=i=>>y10q
zX=^;CKSz^sfX3zOe|`zqYP>_Ocd0{MC-9ff-#unt{Y~i|!;p_J=l>Dh$-V3w$c+DM
zL!`RY%}6iQ|3~v3_5TC){|oiMHc-Ysa!&n!M*Z);8vjMD|D<lXemW9k^~=#nZ$kMB
z|JOFQhZ^l2b<h~kEW~(BjL&yZ7rOnXXXMK=y?2)1Xxqm2ZL{*f<pb^ixzX03{ZF=k
zr2S8Jx^J>DiZBiF+};`FtMfIA=_Qzr%=sM3gVZs3Z>K!y-aay}@tx!PJXFco^U2Dq
z_M{~jp^@Hn^JSZ&fvkHVKVD|9e{5e7*K4L5e?%SXkvL!uAR5tR-rh3ztiUR)!8&Zf
zCTu~n*Zil|=0B}B|A|bghkMC1cOTiTt()0j?V|KqtAE&K4q%lzIw#BlL|L!(9mA#H
zvDPjMuMFuW=0lFMp8u%x*Vl~;XO(?>_*cYhslu?IJcz?+eWsm49>+=4AnxrD_jpa;
zFedKXcW?E-@^40_{w<aNkaEw!25n;ggTvU$$f165AW!3L^taygJUMVFEA*3@`4^81
zLp=ZQ5_hGt?FxAfH*gEp(p1%T`$^I3&q!O(?2k;pkIepm;@p6Qd&>EJI$9pSV|_m!
zazDmXJV(v&ckEAN5BzcOg!<9%ghYY8?~U&_q6y6--!Z@Jop6f%wDrO};_^;tKl6_C
z{SNz|wfN_p;m*JNs&7$ien6M`0Z7yPPWX*A`r-eP$1n`}=(sG5Dr?Z`J@L0s?7vSR
zgFeqm&k{zdu-sS3Z#*VqGKw&;;+-&!9BeBLGst3;U?_|Wv&lJ_hxw?S`F2=9F2WKl
zL$z`Bs!8T0(Cdw}*KKAeuF(hNZmKdjpv%~KuQ7D@m3{OV{omFL*2uc2-FNMvucPle
zuAP)C4X6`(8vN(&{(Ij(O~$<#YL3YNs4I~F^ZX~{{5ND~{pn2Qyf8Ll3$~#Q<=BG?
zBqz!LQ-wcE_()9=KGHX(|EtpfNAkaWPPwO*Y%7&E$&Lx~zh~_i#z7oLJVWXzS#?Cd
zrXQ!DL=EcX^?G?X@kG7(;$`2UDY{*2MjK-LH%X>&8fTHQ7fN5Bj+2hL`*8`o+v5J;
zxf%Q4HSQa@g`vCV!IO7!AA`@$fhS}7e@s5bbA*2>&7gIqv3)WhBQOfD*8hy5$3A~I
z3hCp~H=t}#CiEhXe>ZojpK)rq?}N6nzE7U-gHGQnNoLMhTkrd>_I+3SzIe61a_6$I
zgo(nPj3SJZXQq)eP>h=C>=(0?dl!5^B(D2@XrwnG?%6X?_I4<7?_kPV1ol{%9d+p@
zJt8;EqtC}szH~}1!V)Ztew*Gl|IPZpSO<TgEK_E#;Qwm<&nkNTa?eKQ{GSpw`vGa|
zg*1i)YIf=a?$iGz>yN1a)MasB+{Rx0#1`}Ogs}$eumPK}1=~=Dq&l*zLH$={{Cl|i
zeXRN&nf-tA)bE$oe>jC!{POvLaZkW<;p{;L_M>CC{%;oB1HEUouyL4v6n$IOtsB*`
zdFomm=QnUXH=HB~muZg__!cvzw^FtZehD<Ae1$&aI(g!uF}D5c0e(Y=?f<{;E1`Ut
zx;5yxRhZMjzk09s!wzww*B{b0z-jtfG_f5tvK=(+)PBHuey#W32>s+GbkMJm*U(OX
zW&gXueG9*0|I;6@MP{91sqa6@_n+$ff9U&vpl#zfcX1yN@fc6>9O0Jl|Dp7WxVKYg
z{n=LO6KUx<bN=7C|BC(3eYwIIhI~|>u+JY^MOI@J{j&*wmNSN|E0BKkq~CGUFWJc5
zME<~-+V@u*TbnyF96a{*@cG8Chwm*OZGXjole2&6pXK~y<s0G0GxNf+x!(*&*M2kn
zXz^HMbZ>_vOU-+lYj2wsAruOKaQ=H?JXzV29e#S%eu(W~3CH(JJ6Xk{dH8$UmhXmB
z``$Hn@LQoRYofJ<?}qfW{7`%0_e0H@-w*ZHsnreqckCB`QyKMpq3PLsp)so@oEXb?
z8N*%wrt{TezwoQ!r2Q3tw)59Q^^vdo-_6UP;~cc%|0d^KD4+gkIXg#>3cF@~BkaET
zjZl8PAQbuiw<nDV-zoT7s5thuu&<yne0S_S;WKsN6=l&-$O+$DKPG(s;ctY4+kYb*
z%5x6d(yxRceE5|x&2MMmhjZTuKbmUY*Fxig;#|ycKzg)DGwjHU>r4G-(qIYy*_eYN
zV@~tPxc+ZG8TbEQKrX@(EJHl|a|O8yYp@O-$Mm0$WQPs(p1t}{z4{3BEy(O&won^x
zskYn!?f)qSVH^K4lw%L7ms>}$v>;T_>pwI$yj_2rei%(h?EQAY9&lu0VZmQ+|4%iS
z+CA-L#|7)(>6!NbFl8`z&pGyiGwcIMqYo3^a}Xy{g9Mt<h9qj{8lT@`{ej=tBY}oV
z)*pCA6WP4d`h)e>AFwsFqHV3Y|L7QJ{lRGK58T%^lYIg`d7jU|_mQ#{eW+ZiAMH8i
zXS2d-@+_)%N+<l!)9d%@Lv#1jFQI84`vDq|z!iSia09n+8+UOZ5AhhW?u&CaqHXgj
z_j7~+WkH+rw?X|cz1QI{)&B+B|I)-Ra{_j=`wVk`K1N^^#-I?%onKo2b4;Da-Mdfx
z$iCiRwExw4r|xL~zEJKx(Eh!t{i|)z>3Zxxju*~EOhyqp)b;I$^r`7_&RyL5eg?f5
zne*?au)!g&0ZRAhhDv#?#I@O|KBGTC&O_B@b<YdshB|sa_X0GoXSY`$$F)BV^cvyR
zqHdP<KN9o?G#=CbKNP|uzgdE1Sb<gev{s(+-O7+WBK{59|5e)m>&1Vw_-_|~vX8r2
zd|RG~KU&X;zqq%HcSpPU>kkZU(FWJAj_vHh68&u9uS0rNZrDJUC$hs9a`zVY`TV@F
zjb4T!d-V*>)Bk>?4(Hy33hYPb4Bi3jqUdo?xbC&)WYEix=Z3yn^7&F_v_3*?OB{{<
z6}jPfl%Y7BBx{gBGwP(jm<DUQ<bxJz?*;qKfV9Ux)bQjNcD2>)Ol&+YRqRUaU7g%*
zWcy~eqvg>qRS=S7*WzfGDq#07(Do?^y-x~4JpXl*=d>Y-6spZdj&;CkdR)6!H^n-8
z`gz1N0UF1uqeiN$)LZ@hF5wEU;mh{>8{D^W8=3t#*$eN|@8cmJqo(Q?=lk{Qe_v3i
zURSrGk=}GhJ3>3?sr#NI{3~rf3`0Ifz*_9kb=Cim9%($b>3R?O|Bw0q$!2wV%WCWY
z^!Ho0>ub^5*BgHzJD*4sGtILT&KMM8JUW(Ye=Ib2n%;9l9sZGV0D2MnYPCQ3^^&P}
z`~L8ofuV<ap_nYeY|Oztl$FZ+v41%(g!$Y9=dC3q2kpnOh^!RG5;9}|n{A(Hdi8bV
z7nkkzMz8mc>n^hM)7K!{?xPLAfsD3_b^Ka;^%KZV=uoF`A-AEO-o046V0l(3<1WW5
zyFlE7zL$NVW~25g>QIkgIsYb66889A1@_}04&x|}Be~Z28oGzG&m%QX*^sAPC}961
z|I+z4?t8sIAH7q#a8ejGNT5>RqM59kuK$5HdJ-wbvjO560SWDnxNrBX{dtGRX#bA(
zjpg-di$Cq&^Vs#YM7e2h0sRuL;2H+keI?{dO9N}Hz2*L^?0>i1a~pSY9}n>uPw^bd
z&DvkveP^=gqCBk}@4YFHU(o(K!~WC$U$FoAUphZh_E_5&2EEr~^FYb+F{NP`nU4_|
zg)u0^cud4(3|XsCMDExw-Wx-hMvrq`ipi3AUA>WBWbcLjSz$K!pgjcVkn=DfyRP{@
zkFA+MRvONV#|(Z8un5&xOWFTQ!xDP^Gi$RSm4;>X6=)i(T{ZH((BPf~R`F{+@m^R%
zu0uz|dtn2)3GFS``hRF$I<|0cLz>=)6nas!_`OiO`aN?L-V60)V!N={dPbM^?<n(|
za_m6`_TwN9<0z7gt$$x`{X5ySUbvfuyWO4*WFL3)1?%5&>bmvsXrs5I!#$lF-_N9-
z$^p-5QKz8#!TaI3aMSvmRgdDjeD8(w+s0z~H`1GsKm%HDmV_GqnfV3AYZCMh>*Cv=
zl!Rt_kG1pNE!L9LljvLO-4=RB_oPs>Q2ddJ|4QMa0k6;hIxbE5-D#Y~dGzBFt{|x{
z>_V(#dXSRmGwpwRADUl$sr|3MZ0pj$MF+i8+q3mfX}Bhw8@Pqr=oqj@-rD`U^mrz3
z_pI!2pZ*Yi+u8omyH)#fq<tm$Rh}pfPs!(~u2SILR!*a;LE3R|{YUoH<jzN<u;Ljs
ziN*49X|^#Xc^9#KOrOpPan8>O_w8mg-c^#TUS{*;9(Zc)D>)t$F&Raeh8ZYE-2bP9
ze6|12Z2BC`!+-1kKXL!T`F_6ui?9UC5ZC^#Apd{w|1<mjFsLmtILtc!b?=2$!d!!O
z*nmygf^8^6TKN_C1sOW}Uf9E(+5cy)Z@5x9KvwX3wg1n4`s@9F&gX{R_jC1&^#32}
z|IhUwpr%UyQ(czX|E5cwLT^B$I;aWd!Z?V-IEv#qi5eu3+-UwkGUxl<)Mq}Y|IenL
zCi`CK{|l={9d`<UvHz|5ccJ<hasQiU;k3oDkV3~4^{+baG`(k|`ggtd2mL(y^u^Qa
z=H8|1-<8@Q{NnyMm&hxqcJDRv2C7!SY`fH}CvS1zM&oGlx~!csNj%4yPhH^KsXt=+
zyX*RWJVf~o_9k*y8T;LM>$~aC5$;Ia7>0a|z~H9bFp7+O;*B8-F&+~!8AX_e87Rh3
zTW%;JXQM`+|MmQfTIGkl*?>lR6XN{+IqsW>`B;EOSb}9pu9yGy6T0;wdyt~{7AX6t
z%l~M;F8||{`&w7a|LUrCvICv!pdGzAY~aS;jpalec-)(Qg>cjIUxzw)4SgMY)V<w^
zV|%F<w*Ok?0J*`n>b1t(*`dnyUn}=%Px9OKlszS`*LcKMa8F*>9!`+4EQou8ZE?>w
zl%X7ZP=P_;dOsQ0;T<I7-av=Rqd1O}s6hhFXnmwjL5}j@B*_#y?wMyWkP}YRd&~ps
z###D#^c~VRL2s3|3Tmck|DaC1<6b!ldYoI_xKuf%j4He68x5B}e4Bp1zl8KQHiI^E
z?hEZP=o@t@$1?l>-}7yr`9?_7`_Sxq3r->KmDo1&<=Ez{{r|5C?*?w+Htym+9%AT<
zJ)p?g|9?sjp0qbY{^;<W9{#m-h&c8+jLgRfjKZt)zsAt(+r<rq^znGLZ*fDFxZ?Hv
z_Yd_mmg-lm)vrL~5&aBY@vMn{HyK5kh8ZYE36lCCT`%-A$R7Qa%=~}-<uo$$|L^IS
z;FNn>AL*A|)i1fJpF(zC<afp%{;~bmuKdDmVaBxq?fN)reX^dZ!bbV3oG{0=zOnpA
z8vmZ;SyQYL;8*EcyY|SZ_Na(+=jZcVfT}8e_qF1MMchl!xLCaOg%az<Qy;%|uRYiJ
zufQs-LB|fy=f95LbI9}ey%{#pH=*yD=Zi<LXQr@)-!_z?9E14<VGlVlhW&&bO5}(A
z<Ut(9Q5?rf)I_rM>+t$~1$qM-5!X5xk9wW{67FwCd5QdOU&kapInMX@eYz+4{`chn
zXR%%HI=v6gANu|aeSfr~ZKk*hw}b5TyUM-#g2LHlY&@R-ahhK3Op+@8XX*8a*njrf
zBaq&Yrf2Lw)`>LiWdFe>eyzvk{dw}$?A*|?I44};cMX~SLpJIcm<Mo!`xesE^$U<%
zC=a6ffH^`(%ndrB3_{ywc^4gsXLKc>=wCdOhc61}oZsE_J2K7PN8T3ZUEIe*?A$gs
zJSLxFm9$$kLH?HS;vQJ9?U#@AAJBy7?hF5~yo6yGT428^as)<Uusnn@WTyQ=zTR(B
zTYEfrrFWl9X6D~p-&aJhUZ^~5H~*1dul-ZEnf-)bjHa2&7R0#+u?<whZ#L#&9_C{K
z7GX%6C%rB=43ybZlY4NK{vo*ntFQ*^umPJ8=XY))+qL=P`Oe$uWhh6SL*BDf`Zy+i
z%oJ{^^g-6<8H08<O1*lqPJP%AZP18k|HQT86a6;Ujj!xid$=pG9|zI(fqmhRvHg?X
z?46nQb!?xn*4M?e|Kb_J@r>M4S?YCdjJ6hI-wnpV$;t!T7!}4RCd<Fte}{!x?Vc*v
zj?(Kdc&``i{PdH^*dI1}&#m6`vUlg#dPIFrHY2tx+Q=j_=LhNQXZ9uJK8?(|gmbl1
z`1i5_oaJ{O{kViHxP}`Ts>utt$blJo;WjzwOvk(AeLTctRE~50-qe?Eh>U%YU-eb*
zug&qCUjNAZ-_yst>wlQ6PCw(j-t_(h-k<+}cKw0>qEWa_WcohapWkNIKaQ6+=_4=-
zW02f#{KbFRUEn_;Q@(NUaKD>n{{b{>54CLe?cLW(wk`D^%=PX4znx^w0r}&IG<rh*
zK;p8rc~cs_`f_}#&@;wkBKnG~hb4<J4Kt9G-@BH}pZfDXtL0C9`d*~bhs^leKK0*D
z^&fj*+h+NDyZlXdau*Au1g-MvY_eV6Dc_^4r+3I7ZSrS&a-sC4j@_(oZSf3sb03-R
zGXHOFc9`eh`B;EOs2(T(jkV_>Ju`o~)SO=W3N$H8V%~2c<NW0sbzt0syslUL$wY(r
z%NLFEM^nrvMTKFN->tzqY`_-mUSobh;kb~Lr@HoOe~~@%R*L?5e0MAR(?abpyx#xu
zk@god=da%}hyR+kg?evCmHH3cgk6Sm44t*F13ADpP(cnBTO(k8!2akbEgd8eBQxL6
zw<(`psEua~m)~)mMBi$0K<`!WhF9x>YFta8(i!5L$Y!)5iE3lwRaMR+r`L}*{*BZ0
zvuJ96J2avp&-gda^XtbYT)~&e->z}rz%8^ZFE@VEd$;+<*R+ASH*?n*|F0eWrTGQZ
z%`Y&v-)K%j(}Z`}NB-jet-tj9?cDZQ30>FO|Jb;DkUGHrhcq(#Ukjr}n5WQ+w&CU$
z;MMthned%+Gqh-2@F~KM_>7ciUh<Apj1Nq5-r0eiaANehaJ;~HRoUBNr+Wrd^t00R
zRcrON4`ZD6{^GYC_6>-8{3mXi6X#oXS8<E)KG*y9S-+27ekoLrHs<^4c_TykZ0)2_
zUF!SKdOI9&fBk0Pf3<t!cVMe-`pEZ31HZUe!H;f^3O~FvIvmOR1{?E;@RN=2gdcAm
z8;(628;)lEM)>^B)bPELzZ(v$pAz=t2hT=^?>`zD4(%+mc5SM;0Bqm~+0Y*SR`}Mw
z-wx%6emm?OC<?pA{f)4D>)!|o&pqTlzP<Q&!*{0qx^vI0ha3B~@ZCqF!e`^Yo{85H
z*T1*;8{zW<W5Pjy@}Xzn4BvlN7=Ezb{tUNr!sowc{A;my^*<ezPQR5ezazQ#t09q<
zt1Y3t+o3OKpU{4Nh7Q+`3G*M2=WrfD7(1$dFFQrgm7ach$avb~zm}akpuB5||F`l^
zosv|hX4YeKpAbfy^0xh=@(!KK-OPS0ca(?6*nY{Ki~dngmHT!-*PhDyYG|O}^Nbd<
z;=mt<eJ9Pkd8j>asOt>&j&k4FuZHjBhwx3!v%iBb_xFK!*@4qp;UCkJ>%SWQguH@(
zj+?jz%MQyX3+G?cyXXTa-wwaSJ-Eoe3={q^^xK1N$Td@9%l;jnKv$})zsz`o^#gxS
z|K2F~tgwbLWQQ-fzv}nNM_&yE<WGF35#-(}zZSkm)_O*2n!5h1azI`3b$<QUaU{=u
zHFVF>*BtJ>`Tg-5q3mDc-=UEEKauZ{U;Robn~E{`5BOcw@_(0XBL91`T{-vn$j*(w
zo&ER87Pbgmrj@l8Ov&Cy_TCHOIQgH@fUdE>ll>3Lo~5I*-x(Il{;qc@A%EAm{2$3#
ziozY_M)Dt#zeA39Pbs;K{M+RJM9y{Xf5nH~jy)**JLFdWOUd8#ZvQjcQ}CPFm+1YY
zen0yi?!ML1ihS0~R(!`hl&#gj+id)GV^;Xq%?a7%Yrhh9?r<I<cJteFA}j1|$O_*!
z9{nAWsbGKDSCtk12Vs3i?jOhs2OfMSe9u7>pR;!ybkCuMZ^&a=;RpKNhc9P^BkJ8B
zs(*i^{5`t%4g05#%Kou?euCqhM`fSrdL#VwLu+w%PR%|!f!%Q`J0$;~wX2^`jLfcG
zI4V1yPt-C&JW?~WFP6;A_WEUYy6y=y#V)*SXYnuauW-Tjw~#!cZQ#G`gsP+LPh8V9
zExVcLKjqh*r;NHPJ>0Vg%?o?aTu`^(`9^lCKwX=q-X5^W6`J+yTg(;hu>P~ty=_nI
zLG!@6&{A#3GtSW?YiHUw@}fB=t|iEZb9v#X{44YF!g0?yG0VJyae3iq+$YJZ(e}IX
z4nOky_sGTg@y)#Olj*s}$+N?;t9jw?^V^G)-g(+@#;M@yHu`7&yMK#+hyDKZZ2$io
z_`dXikh@7<`~%m%gCF3hsEfGv=hE4)za@R6Klg8DUu>S1-9MhZMUKE4ESZ*lu_Efz
zvoDUCp51>X%HPlK-!MM=;-$Zy-9L>!k$-+k_Qf^tXZMeJKl|d0C`+?1p8mb;{y8P|
zS=s%?zn6WnhI<eB^lxVOZ}Xhl<YaOfd459n#fiU_-Jihh;_Uvzld><Ke>c0oaANkw
z5fiif%m0?&|1H<~r}!t_^VIWi6=z@ERGj_S#o<5rJ}ag1J-9gjxBd2avg`O2`Ck1e
zz3*}N7ynlFALnuxzbpL+Z^W$Zi!0vC?)Nd%rN#mu6lMQ2*L#$yxxP_X>2GHLsW1%1
zl~wUK4WjH6`3KUC=7{s2L)m6}Q?K*vj{J@6hN}GV54h{k{bqKK>t|j6Yvd)8S*&ad
zeK{(##$+Go_f6rB!%vpyhaWG@H$Nml{B`ahEuE0vM7E#}N&Ghc1efsn)Zfbfr{p^S
z^&jA~y!`Opjrn2U>in>k`#W>TXRq-7duRS`_WwfuJ#6E@f&4=-b(IC*WrydMtwfOj
ztt0w0<c3`S<s;<(QT7|VJuK`lv1eRSUf6pwFMK;}3Bz2kn5Yd-j=~5el_dwx{!#ee
zy#F(NK7VT{<kxQw+xH(14@m{t{?o(5M1J4n?tiKbZ{yGAI62-|DBo!p%!vAJy7osg
zy##+7e-||s!$R%tKME`9^;e?3(|vybzj61gxBq}_Df^?4)CNhiqx3)5Zt?Da7_V!S
z{2yd|M)y%?WG=I(RM-}JAAb_kPq&A@=YKzx4SzR#N5Q+<pJu(A{jI!rv&+Z6o4s?~
zyV<)&zMH*!^t;)6ro5ZIcj~*@-=6Sp_ID<|n_V&M-Rym(?`D5@`n%bm&6FP1jR)?%
z6}}ap*YDlNhVV($Z<rf&$NY<#-pl)b7X7?ie3#FoKF#$gFJ*@xkw1z09(#tGuTizw
zzIMys3Nd~)weM!vHoTi{&y7&&o*3TG;=K#>-`hRE$h?8*HrCf_en*@69e>K*G2HwQ
z#Ph?G3d`hN-=<jKVai+KpSqqlccg!by(RCM+i_R;z2*{Jh;_a4^`v}R_M6!kx5;w{
z<@4Mr*%$B2GppQp$9-qi30LjUfFHEj8&dq5gxQKt{C|ACdstl6*|7hx025+}!2}f*
zn~-2c#RL@_tTfWMu|~xjYgDZGHnwStZ}DwfNyUa_7#LuH;d~lkU<PIe1{lt#If1dJ
zH8$9wL4z$;L{!w6#EKOw>F=J{_PT!G^<Cc|_w}sPUVHDg_BuanZGRegkZ(Y9+PBo7
zk>h@M8nEN^8U5eE@38;D?=~&+N914q=1JB8{PG;(j^Co*_nWE6kAHBB`p9o44IKM;
z7rz^M>z}h9Ne*Q!7b-){=lJJ#M)!z>Z{nMXKsP3aIk-j2H+nk06y|y2SF;uzDsFJW
zC}@1EB%8i~Hatjjj;*5q6)d_L^rEI#q5p*aA-;e57KNO*T@<=fio;flf%CdchPJ5E
z`BhSuwvxW(O8%+4QYzD_uh_p(9l26!bX<%1IAdpH;!3Hfuh+o!ja+Zy`r~)ww-z1K
zeXGPmU#SH-8oWZReL=i0>%Yj+Q1tV&vGd(@7&#+6mHr~n;dI~^jebD<O7sg?O5Zuc
zJWpTTuu=wNu2VxBSE1j!N=9IG<Hh2NUMZu@Pk87Pc|#+WZ-(z8?qV4S4euI3dNo@3
ze>HUjQ^y)zM2O@WL--GHh!o6*h(0=6DeA1N;3i0dhu~*W=3FJ^yModG3YN;f!BPd)
zm}~Y0OKn)N)W!SXKUnIK4WaC*2uBx=wyz1<974P32ocLN?c^fuWB~0VZKI8Fc4QmZ
zwIdzaJCL0bw5$3M-rs*dGyei!4w2q~5b4X}eB}(<X|5kk3qg01cAD@bTtA9*%|<IX
zZXWPXMk`-p2>WwF*vA?oIopE8@+|*R?mHyU@jh0}w!7)$VE!8v9So8g+T3*kv@drD
z$<@dWuo)PFYHkF&mzsWIE$1u#xrmuzm}dB0)_2aXkoPcu1n##<|3>Cr3Yd3k=f6(4
z4IDWnt;d685%ZUjBSG>{q>l9z+Xm`m;wrc=NJ5Zn;Bx4R<=I|DM`R;&J~1nVX+RBI
zM`;Hb!|q!l?;s;-XZ{NwgWo_2yhgZ>A(*h&09_1?mv__7+@NyiZ%~?L%E8hN>O#Jg
z_qnsH7zgp)lio#80D36Vg~)fgws@9uU<i>4*A1#NX^Sc|-k{2lUMw}7`%#;*MO7UQ
zmFmgOsvdj8^!2I^Y0O+9k8tf%(71fDWFgDob1?I6R%nO0pD@-#_5vzm=o(QD!e}=z
zd*OL_5nh7hZ~{)kDfkP#4e!DSK$l;07CwV7;45I*pwTuSk|3mc@gZfIen_<}JfvC|
zGsJp2gR^fBh`suNwD}I8AACR@F<GpSWHEo5C0&tOjQz9F|I9)^EsOmnS<<)b5c3;{
zm|s4`{LUffcMi#L+94T%(Ue2t0(Z<I%HJXJZagGD7>hV0<4{3co<O_&1~QpCj%5|<
zGW4^kD_h97F6e>4O*B>Di+)zTLyxN8$59q<XDinAxWO@)hFLfbi?9qKH_-2Z4A4O}
z*kA~zK$Ce`va=717II97CD(9RbbW^<5Ava4_OR%o5Q>%$OY!_+F`PauCCJj5!%}8D
zEalF_Qh}`W9hNF&HTIhHLsA=iNb155iIMlLUpOQU(71GnvH2ls!rVN6NX(Ik#1ef-
zTA(%YkXVs6uCpWCQV&V{zC+?5t`1};>Dq;KV(&)w;NKg4i2WC&KXQQU2a!Xh^Ki;<
zWMnEsMwc_iwU8n1r3~>*XNniI5BIUj3>lx!pdcKO?8F12-E}~6b{>%2qywT$J|KBZ
z&q#jO0meVfYhW)-IlzC=pOa$38BEVfNgwC-L_Q~F<qxa!m;+LgeLyNF4zOSRfK)FW
zpd1{K+Q}4EXE`9oeUGX7a|fg$`7zZvc|c5ipOdEShgI{`0WmZFuoxawE%A@2){p~Y
z-FSfa=ljz?rrL~;sdm?6%8_wEI#Pcl)8y9-gx$zD1G`{1?18<I1=(N(6WCw~cE^fl
zAK1VJN8l(-fiYv1e1Y726ZIH80FT3u;3uH{_&Le>$#3N#W)0*$^IOUH{#Lp$kHYit
z5}bnf;S5~0m9YhEfr?)qlpBz@K>2+K<y**m;b-ty_#1Q{KPc_*9u&u44$9xL|7M$L
zeha10^87)mL^ePx9KUmynvh-U_*eHZ?zmTV;x+=OVHv_G7bQ>+9njx(PzLTi$XYJ*
zDeyA90)xLeC__O9DG)0p+q6Qo$Q%c{@GknlkVnTq9|{&$h#m@;82>^sW&@P4hF%I~
zbkxhCg7Ir5RHX#)-wXbOfw>myG8z9uJ>~{z#Lomxn47`uLf0EwjxqiP>-5)qm+Y9^
zpnZuo3h2Px3I07xPUyzm1HCThSD}9w>)SAx!u%=>WA^W_{LlVO%pUOK=YuiK<M1kJ
z@jAQ-6*nW{J@^nlfluK}I1iWJLY)J*!2x&<w2%iSPyxf>hS%XucnjWv&*92jX?q|F
z?t(lhgbr}S3-AtH6wka0JP1F9U%_+088DhWD1;WU!wc{-yb7<wJMbZV0-wT{a2}d&
z<6D3x-D-Ic`5}A?8s^ZmL3=Ea{-6K63jWyxd6@H|U@1`aP>8t*it#f*2@P*4>u+TY
zsLJ_YO9ktUl~9$khWuYc{?j*xy394?{~Gdt4f)SGUHF-x33D@;U7RBZEyq}k1nV@q
z*I>uo2JK6%k3t9LPUynV3Eh}`px4FvCiL%OeG>*#Sl@(U%p)+G$@(a`F?+y^pAW_`
zkNfSJLE?ikeUOZs=vQV0q0bk@dPb0VU~qqs48bsrKri$`KMX(@VVuy7xd+;lg2VwG
z&<R!({kgqCVuv;`W4Gw&?-}UtAsbWa5ALHsm`;BXS%=-Ylm22d{Y7L|B>hR;Yw)i{
zmSHasroS0Ve-l|8M|p%2D1`#hLm?DFF6bZ+@<9XHpoJW`_M3dua2xyz4!|MMLLT(P
zFpR;g@H%`7U&48a+Af;!z=QBGJOPDJ0!{D&yaI2*=WrfW0^c~Sg3I7ah=Om!ci=8~
z0-l7Y;1Fmb4@#f{YM>sPpc{S<FT?BbCcFjjz=!Y&d<y5`BKiz#;8M5}G%m*fpgqR?
zALLH6KM3+L=R?5~V?8LuTm;4V8K4AnDU`XG|APwV0V<&?PNW*HA?_%+9&Uz2`Y!uu
zi_#!^Gw<^wB}RWo8U3Pc&Qj36dy(W61WE46t3+3IrQ}_~dIi@Mym*c1WvvvpbKYE6
zm=w!(Vi>$ZN=C1jQg00ZhhT2{8P36D+&9O(u=W@1tLHi+{iw0;;!a;m`*ZHaxZ8vK
z8OG>-ooa0^I%JF?=jb5L(K#H)Z;pK$zw=AR$$#)m*5Ur>zlJ&dKlvs0Gk36N^<tpB
z=C_|@FAJUOYxoY{@Q25Ifp-Jt&wl$T;U9<l0_Cs%@ONYWj4+tr_uFs9%zVFpZ}&%j
zJNymi27~19$bY~;;mj|BB#<;)0cRc!k~0qk$;Fs2gDb&DGR}PwBrhQ2uwMf#b!g@q
zgJfm2X#NMf!GGPG$e?R6-w`A+NKMGSD(~vwTu3V|={WX)l5gW~45Qr&C6BvEQ##6q
zpVMX?QBrge$=YC33~k6t<}jGQFs9PxA)EHl2Kw#HCzm6~?pD$~s^l)>GY#UOOdHty
zb$BE8v0U67%=H#4d6F<z4Sv|4!ffGME%)wu7fD#{<M=W6;J>H#AP9AhmFt&q`z3Qe
zR|iRx4l`@k{lq&=|7@&_@n1}cTsWVRHC}BoW55rBM9W;zueitPS*~F{*qF{9t-Z)h
z){TkVZ}#7B>`v~9e-jgT7rs~9zx;jJuf~5p{;aEv-ACL|KSNk(K6;VNt;WrZ{}TQ`
zVZGur{LkUf`b;x4O(UlW7ett7=EpY@=d+j-zjo7pm+-jz-|b8z&lF7f@QcL1KeRrB
zXT`4NTEG7sG~w19O1uGIKjYl3#D(2|X0^XeoH>g<o@>hZ2KnxrxL+=5&~S7G>55L_
zLEM<fZ^rKbhD{H!jyg#_>A&_Q*G^z2jQ@VdnH9Vz*9iCiIoCOf!^L%&{bBs??$3Lp
zgKHk;`WFdfB)mT^z6Wg;b_05xm^~t-J35rCCpGOyZvwmaA?DtLD8IxvKp8XkG57yV
z+GyT)WIN$3tZChfKhM(19G2gH=5hc1&>2`uIG)|Pp6hfxR~l)+Gx+n=|Bn8;z;5Ol
z%LtF(9N)A54E#9;oO28$=NL3zs80->Nx6;ZJ^uAIM$URX!*ic`i1PnS*bLVZmN=Vy
zgyo$t<R9zj7p}K)k6Upg{f*3r`San-SA?Zr^1r`7P0L7s|NdTv+ZX)nus22e)#1^b
zi@yFs+WG4-|F`4+hPv@T^E`17pY{sk-p)Np10(frJ@uT?UnZYHSDA3l(WDdc8W(UI
zWDblvtbT%LKE?HvZSAQ*SteXIdE@`b&NsIBpM`KdPcClS>(EJAtK@pz3+~{Wjl%aO
z@*MUv*U|UkK4)$utqZt+2KNsasi0o0K0&(2VxJT#A?(;o=r8`9vp%WkwD>nA5l;+y
z0r+Wizs`rTmE0HmMc47%yj#<0(%=N~?*BSoe>wC2m)XcOoFOmQ6RwAER2zhD;eA(2
zuUySJsO)jV+>}cEd%52Gb@(N|r~TB0{=CaT`tG<|v?!{KANhu4`s>KS$glH@J@#{F
zBQLZQS-4-wSJLl7{_);=zneb~`5ydvwBWB#|HAY0JuO{9Ja<IO#w!_vU=E0&jl&!R
zu@DFGkN{__N)r8YbEG68`;LkP6Mg{MKf~Dn7<;sius4f)3?uz#JB)&7f;nvP5ylJS
zgdZbp2-k)}GVFyg2!}K4pA9@CcL&a%N)DXe-mc~z&kUR!d?3(?UmpyC3oiWMlXW@m
z4DEk9d5_fXC+`E$hoi4m5Y7B&JoEpF!BRvYteAN=1O2cP%%yx!Wkb~UKH7g|WisbX
z@8KIjAEpLbi@h#`?*V<6dSnCkM%+y)d<VM%>Hh}O|3$X|T8^O0GQ)lt`qVc1)^=pu
zGWx0XxgF3k$vii7#i6?b-OvNQyV0M6e(?Vbb8tWUH82b#FuEK49B_jN{{7#sf9v7-
zNP{DA6pq0JOu`gQ!wk&A*$qnOkqdAd&cPxyF5V`l<=bQlb9tGPWuyak!Y;ogkKJjf
zWEV0SM*D6P7wpEo2T~vvb`y3lav$u$oC2w^57HnV_Cp3_LKb9$4)kDva;OG>oohs<
z5!YGF=|~fH3)mn4doYwg<NvqTw^Rt`xl>Up6d4BL5CM^3V-9H#^8*g#5pY5cW4SXg
z?NCFQXRH3EX6s&6b5(y=b8COEW~+wPY?(>T<+iA~mH8@y`$fW;(E)X4dyz76&D+Fx
zW<#wy^Y+_H`>v8S>?@gzb03|SHAsrP=1L&RHP^z8a5HR&Z^L)t9(Vv|HP5QE^d0Bc
zKc%#ql`?C-M9r>T$Nkr+xht+w4-@9rK<;<zx73+!-%z(>zSD1CeW|(&8O^iB09`cA
zgAj{34ro?1Prw1tLIIRO71RUOndT=DPuK)V$NV_=O2nK5RF4{}1`XAa<~gW?C!qqK
z2C4zgNzkwco}Eg&yodfT?NM$fIvJ3MIUfpAE)u<k@r;e}jBkY$BMr2-CCE>?{$qF>
zDniyvCHuCk&awUuHBbw6leB?QpRisUkd3VCo1iI+wmxz_bv1Jo(2&R+1(+~5!baY~
z-=8a|KT|e&kyJ)g#-SR&DyWTQjsx`Pc*cc`qzGBKbdebP(AC0T>bOWsT)Z3Zn$x@+
zQj5QCl6S;EADK7pKRdyHh8ktlM;QelW-p9W7mPt4=3W@UuOEgm55kCvG7kQK2e$Vy
zH<G~|3DOzP90~NyP)C4&ZpBQWvQ^I<3;zE9!&%FjTfy%CKio~^+Y6-MEASq?2|>8^
zBENtr%&VXYvj&ENCQWl5@_?pL(}AShp(#dQhP)a=_@+N%F6?FIGjdnZwgpqJU^CnT
z-+_DK?2Y_G7MTh^g$kIv`{}?2<m`P<2U?N*b0@G3*#S<t>vrZo?qjUChjHuAxo$sY
z@;Szr#f(9ld9HSn+|9jSWSwS$`@hb7*I)UrJ?O1;^DH*eEKv^i%q3co^N<ury#V*~
zyi<(PE~1PVGOl?NdnEP>WD{<eZl`YIIvs9b68_o*{INTs3A>kfK89ZkY{z^fl$$S-
zPq;S1ABH$H@S7lgdU==GlRE-uAH6bg?k`&d`Ttko?D|gwXYYJ1aPH0z;dg-pgc*i2
z&wdnm=J8JhU6|+n_V2zGc&08oaCWdLaQ2Scz`27hfpc}3w<XE!16_f0S09$Sg1h7c
z!hYlrzx}sz=Itc;1T)8ZF%B#YJhLG`P;=x8$)34Fw5*NgK<>gC(Lo;ThxzBOp#8f<
z^rx?s!uhMDXk(-lCq#+?b4l7-DV@4f%5+yrc{1mn@4Q4RS+lJ|R%5S0)@Jb!KBN(Q
zJ+dL~D*k(WiI|`Xn!$YJ60txFw1U-jvDm=wxLDdIIG=x!{|_#+{~y_jy=$8N|ImGm
z{r}MGV*fw%r?LMZ2KTc6ABGcH|Ao;o_Wy&Me&GL2KmIe);_vV!EQ3&7H^6KV-#hl#
zU=8+L;a>PT9EKX8cxgU|O9*=@-+0nW>Sx%2Ibk#NDbNZpf$Lh<Y~Uq04kuvib<~AW
z1NATte}Fkyge6!8_rhxuPXB8g+zW}scMm)QKY;^~1J%$5BXAsEhj-x<_y??xrhLNx
z!U5O}nh56qKpVsSALJ%5{|9-P^PvDgJrt%f{|CjH%>O|N=29rbuN*4qFIC3U|B0vn
zLtS1&onDKqOJ}T>@h|-+WFvM{JmWORdCf@k1bTW?=s!`<wj!-d^nbv<K>r8Yr>Wzp
zH~o9iJE1E9-D$Y+ACE-p#+}rW$zPu>I*>};i2XnR7s;Y--9`TgF6{ryLO*&R`hq-z
z{|r?P@9zlb8w=pu;v4M%C*Nro^jO&EfTEiv<6<%Iy;xdLUo0(rleYPb#R_eFTlSQd
z;v(EAgA@<aO?V&D>k5@P;vU1^hrM??l)Vk1(vN)zIf(xVav1w|!rcu&gnsZsIauHX
zWS<VDG7FWQWzHx89p+rfC(gW$S5US>r4AZc<F98u-;^1Oj%BDcr-d?}43%;{XLn?W
zN~JNBH5%3=u~#8$vqGf?`8MCh2JT~k<M0-&<lBse@4#8?U&3n45fBIW!=rEj?t!1e
z)1ZN3sDf5_3mTyx+Q0`V;T;IrLSGB6g6~5Gyze*jeW$`+*av3`dlT;8hPz=C`~ZFo
zhv1h`124cau)}LG1A}k{<!2RqggF}dDe^101@oP74?F}<z^~vi<UteIpdbDSQ}8|n
z@JwqU0ye>|@GbZO{EoWdy&&3#VAg|R-9^*`@Br}1X>y?f<_O17Qxi_u>)=~(KkS8P
zARp>Mh~pBt24dk3`0+~0EgT^pmfJOvA^!Seo!o-F9`{&?^M}FjFSI=`Z=)YiTjN3Q
zq<z{#xHBKGlZ%jl;nVsPVRQV~5dJ>+ig;IHuY&6d69R7&?qm2fZm3~s!U>m$+=Kil
z;qA!s&Eyqf-obqfG{U10&h?++K96}lG{XmQIUIu)_!zE&<6wus!!n>MpeaPAAOiul
z1x*s{g4^H<_=31UBTOvj)$lI%SFx8sD%Who?GpF^_bJ@U;YYYTkv~Sx-F+#2XZT{B
z%#Fgwn9cADc=l2`^L!|C2Jj5#2cRAP0;Ktw4eNxZFb%)w(X1gpR9iHsFr%iW*@$@s
zem_I5Mjk-kiu@*YU>*ikSr~V$<$nL-nK0%|Ft3Dn;j^`J#tU!5ao7%P;g|p7KEmYk
z%pbru%=pdy2=kp_g+ZS2CGg8SI0%o!eXt%rSWDe<hh(!?UAugT<REjg>%tQyFEUZ`
zkp=ueT_2yw_%~6Ckj2;yxR>BxiY!Y{l=6&3smMx{N@NxGYTRq^uSM1o-WZ=q-#Sqm
zkd4?)xHsY7j5ME46w6|wv@9n|E7FSHhPxgAHe@^D9r5fV=lV`$7j`G^-T3z)d(-*9
zdj|h^&*J~?$U*EwxDSUV$_R|cC5j8&-~n$K|DA_17zYjichBbCwY+;y=nlyZ-yyol
z9g>I4$6gS>L-dI|q!3wzy%=``{w2s#>}44{q@1$^Dv*`ft8lN5*da9`JERus5_X6Y
z>M=J!BYq}m!rTnzr5$2{7R;?+#m@$I%x%z~utOZsfw>d9@N+^p<{s$H<a`6@$2<Uo
z_zl4@<`Ec8;CuvdWA=a-KOc-?9tX`5?+@BtiIM}kDT$(kJk0q}kV*bSA?6||#?JsH
zm`kB7f&7OH?SC?7w(Yry`48HEWG(i(4BCI%UH_bI1NKJTP53t<o3WdRX#ah*|H#$}
z+W#r`PR(2-{ykJ}*xMJ`AH7WbkL<+Wg}W2~Ze$Pk-XZ2jx&Ffb@?`cm?O|>-6@7o?
z2=>to##mYCKqEcay}0}EA485~?;`J<Hqz5cdLrAgJNA*@>7+N(N?zM`k{`+B2hxn)
z5=?%P*R4peiS)FQo?JhU9L4U^k=_Q<8#y>be&9Zg{|K@dd*4y=Yl8ej7Vo=V4C%K^
z39>Znb}86-JNg#4OChoddoJ!e{PU3c*fqho3;LR(Mdn~{Ox-S~dA_I9d{4+a?8cqj
zSwGw^4ah3jcGeHKvwpap^~3E_#&zXfUxBO)NRa2LgHFL)a25_wKSfbrG9|5Hn5=mS
zvkCb}@WG4JZGWL{`YmnID-eQxE8GP?hUcIh-k^Ou3n7G!f_S(aeh5!PHdMivfEt|U
z3fKa-!A`gz9)TZ08dO6Qr1Fdpuc1wV+ewbDRkSe|a~<YyLnHI2%=HNCg3=6gtLew!
z{v6}3bF_gi$j4aAKgyWT#W<{fTfBT6OFIf5z#HgaH0Q?4tBlu7-;I$c;|aq$`RlNX
zYp-GacYt<O5l1)NMYt#Ny9@VUF&11PUIXzK66RI>Yw&vk_bS@r=dkBtzmj+^=2^yw
zdr)(w4CO}1aKU>rQvB|Px!k@g{ue4&$Ux&&=<9wgKI<9Qbv}`C>uDL4x5dR_B5n@G
z@o-p^^?4DStQPxAte3M1;vM2&!8-(Ve<zbPU925C-(4r&A6zOuKMIrfv#Z2$b%=C)
zahY^}wO+2|nXcg-uIKq<7@r=2Sf2maJfD_0UcvuU{7Z<}4lncUpAfc%cN&A67#BYc
zUh?Hy-sdjDJcs}NgtL&In|PMD@Mj%Tqlo7=%=P%aM;H_Szx1aI_qmDtJPDT(?%QDZ
zKM&8*$ozE&_6eTr7}Rt9_1K@_KHY?QinuP}Ssz1AlJ?&vzTa}so4Hp%BooKCxh8>Y
z{~Px{!hFL0in!)d;_)kdW`j0itK=**{+Va|bDHrFGC%7kDM-3W^c$5FX0rb&ntuzj
zm(YML!L2lsHH>}eKQnJ!fvn6zzZ+SNy#`r}KkEF<G5OcB&~xV6Mx+UQ6S5h9^E~=b
zr#bTw*}9DWGt&R>9y_uv8U5!B_9tec|H>RrC;Xr1L;UA_2gbJ-{x6joCI8<4J8~NR
zmnrrq&agib`JeM=nG+f#u5qxCJ_hpPJZbe)()8P;qnq@3557SfU4z>WWH<i7q+vE3
zg@^G!Pr80i8s5V-kC7Mg#I*pIlg@2ibBO%>4q<BWdya4A6?hL8@UwIMCtNp2evK3U
z&+tFEKg~66c#8X#aP7^+UjuIu_GzBwBH}s(3p_(O_gaPy!heomAaVQ}zDYd&#Q!GO
zUrRjUgiYZc+qu_a{J(?S6Y#V@|97+gAHZ5uD)aw)SpQFE{y&5D|18%3kp<ZGxEGpk
zkfIdU|97$e4<)2wDU=;!ju0xQng53>%+*k{#2g{iVKzcNehtuw*#u24^q;`Ii#(`i
zZf+-YbUcF<Y2%)DWE;=Wj&xw}$YhL~ME-*lx}gVpSvTl|ei(p37=mFK0sp@_T;TsV
zhX=gigE1IC#@-3i;7QW+9rzS(B7b5@FD+?zKlYcQmNYv@x!H~T-K62qp_VlMJ?Z`m
z>G&LJe<k6jace<_k|)ume;iy&n0w%U!gav)#Jh%j-p+kKBHn(&ClPNkao$Y0cgW}O
z<K7GjT<^yJ9RAb9`7B{x;2z<)KM9}1J;XT%j}hmW{_lh9U+0=s!rxDPGsu^bPZOpP
z|1tbJxc966_vHG%giX>LM%smwhA_nZ#UPBpFu1}=L-2qb{Qoxeg8$!!V_;!iZ7$~=
zJx3_}L^(IX6^agfsI+0XBOUm+<9^{k#7Ji*b@G1JZqk^aMV3b}J`ZDj9?SR~S)IuE
z9BQ%GfH9KsJv5{<zK09r`)qWl(!L(wXA{3)F*b+$X^CIQ{d4=6AHbf6)Z<@(dlBIa
zk%mmprPv?J+)XI^W<uq9(&qQ1-S5el58+DEsRg&6l8&LI`%R?TB61%7M*6QK{XZw%
zt(Y$&+&uOV3EM%sUP7MjC0smZO%GG}O=6EC&iCLRuKOuG1&82EuK5I6!95-(j#b!i
zgZs&w2jO9G!%evT9^!~=52Qgf`5FiNA)0$7!7k82IXuJj74bX+;DPcl*2-(h!#q<D
z%)n#hzb1zA584FEKjdQ8L0$^w9}1?g6MZ`6KPFm=_EP@A0N3#Dw?kP3`WncJaOza#
zTTmTO`A?+$BkMvb|M=BMQvQ*R*iDO+|C#ILLDIw=93VH71}y;r@-Xs2uo0&{BtRa+
z`~)0;4&v)1zOLA>=lHvcuZQ@0dA7a`^dGaR|B-{(hj1Upe*`&7cvn32Ki7MZ-szjf
z$NP<Ok8yBkULzhDo!}fB<XH7J=)ti!ISCyEWIua&2X<bAo;Uw7XD{+_D*r9VK8PG*
zPw&6~FW!!yBlH@1j%RIxfB#>+h4a%}!3-9#L)#MV9MT5$?1gUNo^?p0?iy**Un9-X
zh~0#}BAETdi|nHcp{y7Zq^9p0#+8gSSR=0vxLOK*%u6OQPpM;`(!{(Z=F(lvQyxnY
z1M8jIaP}M1&T4{{WFzw?XxIB_=dtUKQtp?~mtjJ@57|4*o(UAnhr_4`BB%$jd*Uhg
z;EJXm0RKP7$H4y&axb(pY1{(-Jsf6eGqHCAY~a`Z>tw>dqmMlt(Obm-FU=k!dpF91
z#fj{)pu-n{E+6+U&q$E6MRfeI*F>Q67l!U1vVnaX^$EdZj6*jN*&K<kAo%x?G(wJ^
zbDOe*B^&9V*w1J3DGzjXm~*qQl_JcApdVr_8%i;kfC2wvc!j*Wo$~)3co+`BE1)Qo
zPr;>>%j@9@coOW;0gCeYCR{`LybW?FTV)?o&bih_Tj8UOX8Fr0?1OUHNLfvUFv{mf
zNCYFi3uj?9W%g>gn7G!#f5DHT5;~y)df*IvkMjEi_#xy#2xU1Eo}m0a40&)F@Ai&g
zQqG=)L(l@hhnL|)&=@HH)s%k|<sX@gT{lGe_fh_l1=#gRBBjuX{vWbfNBPIC1X;RF
z`;RP-h?I(Q)*mrf5l^-LW~oVzklMtnq>eQ=Bh<%UEe*uin0U3A9F%|JYX<XtB>SIe
z|M$`UXVLy6ZP@MSBBhObrrpB)H~jzpe@8ssPTCL~^WS?Tq>nlB{%OiT{no)nzFqpU
z!^jcrqem(KlYGN_BiZN6H%mBQ;^i`Sly7-5K)xgm){suC{NDi#!+UTI`FKCv%s22D
zyaJbz?@{m_xC<VHr{O)gjBnu{cnXGL4d2A|PzYQ2E~4Nen2jN?_&!46b?kv3(O-wl
z7^7Va-(wu{AnbvA;C^_7G3dE9<R!chp98<d(e(42_g+F?LN;uL$KYr1zhHtFU=)1N
z0ex_uYXgxhL6b!O$5Brtlm9!(|5WlHnTI`pBlSw^H$+dqTv-3uPyL_4`o|vfKZWsn
z3i(f6S`HNn^#7m=b2ZeE&KLH7FOdJhGJ_@xu7{i94oHH#-~reJsqhs13Vs6zf!}&*
z^iT<PU<Ml;g>S-@@FbK#D9l_&e!v2pfjRgX-iCFTlg_Y+`BPvBsL`O8mJR>FoWr_$
z?hxmr5+<(?ePJjlN1qr9p$Ll4vHlJvPzq(`=>J!v|IZ{rCGk}0Sbt~YpayC&*ST1K
zXX2oqcpLgyf9D=1^y2*baLrs}-n&99`&Mu^JO2VsWB*?U`_J~T5IeMCZr{cEz{#A!
zzmxNUli2f>%HFRO&IiW52lw7M&i9SyeBVU&pCxeiPc-NIZsdGH7{NRmfo@$SXA_09
z|16CCXQAvr3qcp}96Cg_E1D4UKa}zxN&ZKW|I;Em^o#PQIFAS&t^&g830Jr*?0?{V
z#RbmspzSG{XHV-a?UIT8zeaSwoap~L(ElBxUFs9sDUn*tb$WEr4Cry`L>jU=|AO!)
z!Z+dGjJr8Q#Im2gPFd`4%@nbwJ)msq4=DRy_PL_>)Skkg&s4_1dqg^SbIwRI|Hs+I
z|8Wv2{|S`;m<Lo}>;sD9KQt@gJoy-kTm=y@{rH3G8^~ygg6rT0xD{@KRtxvG@eJiW
zOEvf8ne9Bg&BwF2xF^rp!LvGe_IBQ(2f8skaqGfwU>UuwCC=DBhfdofdRs>HwoK@8
z!60Uj6TK}5dR*LN6fT@cL%rtz_qaEWGd+2RN8nDNTGHGN_rvWledBI*FY+PC<h^G3
zzT?(Xw?Gs`LS>T37UYev4r)_Kd(x^JS(Qqf?<eik$b)_41vEp`Uh*LB2dXihXJR7C
z1#a+w7j|>MF+-5N$NhR|=o=&lN&imt{ZrWwfE>a;jQj}yb_;F$=_KiJ($?E((|xq<
z$ZqUCNV9{!L3t8uhj)tq{|Z+4_x}~zF#nS{uHd;Jg;Zc!U1Qo4EQgWJDYVtdN@()S
z;D7B0K-PrPS3r*9{zo_i8^fqW;U-9g`#`gwz5!?<2Xc4QH-Nm<VA+j-3LJ!1=z%}M
zNtlLpml7Y`4EMl`aOFDI&R{#-3oIvSo`zpTJ+#0GyajWRxSsX`z6U>m|AwEyFW_0I
zfLdq<D>$GV24EEa2q)lUxG0>pO1Kfe3%lT9_zj?csTqXl;V4YNC$N(5=5n|d5+Dig
z0)7v#c>**}%0KO?)<^kA=3>`PQUA|S|04@d)BZ0~|1Z=2Ba5*ca4*5X6j?_2@*(Pf
zuCGK^VXxkU{$DEkf5^IY^#5ox>a)=QLpEYJ;ogLQGt%s23^|1UpO3yI(u&<S#rk&y
z^Le!E?dj-0Y-In-Ec$=Y6~q3QdB%?T_dsto<9`$Fe*pVm%4z>2+5ZyE{udYRKa3`@
z|Ha_%!?OP+jQuXzlsy>Br2h}Oq(K=pLMOZg6YvT7|4%rGnc}V~0;&g1Gf<p01At0~
z=1U0T+slCGAQ$H0L%5vlZ-MRb9oS8Ngc9GSa6MSS4r6eZctW^F4_6cR4ZgJ<gx`wa
zkKuRlJ=}kSdo%Py7~cdcT$&g#@U2zAB)ksGpm0wk+|w`uUU-2x{s1-50M<m(dnf6g
zjJ^TVjNKAVe#DX=$gV8(EphM0zX#cl-La4USUUNE93=eEQTkgGq&Kn`dtWs9!Sw@3
z?=oWr-2Z)E;V5?3Jo#~&IR#{H@M_V8u9iGxe&lLiWVK{FR*N=bwdCw(tj-w~1}K42
zD2Q1t`ta3KNZ2A|RpM%?CjJ`kUyCdYSpD^WEMz4#az7L9P53t>>#!T=`5w7{15%R~
zB-x<dA4L5SB)O&_(dmLDkNPhkS%4zC9&;h)BK(Sx1_!!rOF>c!W%?j=41(zY21zAU
z^##%X1u_4{`j0b6>Z*gp$YOPU8tXsMn92GNG@<|2%pPcSDC<9w=vG9q{<9O^ieT1%
zlF|Q(XZ>f0Jq}^4|3C+H#<SllHb|TSL7Yt(Bt4O=|4gv{lg|222It!m*C1oVA>4<N
zBiKiY$Cb$X57L9(i@Oj1G2}RQ%_9AuW%TrsIkaE7r)mEg|K&kG6vQ#EghD8SVjcaT
zaK@RDj5CpC>Hcvk{U6*bP5<iuV6VYm>!bgJG-9tuHsIfgG+}Q-HtT3}vuJy>`QD%v
ztY8B>v_U)l7Ke`S&A|7D>@x8^TKFEJ2YNH;|3E+er2!rL|0jZFXp#N@%Zz1@QT~_s
z-oQ;;<ADqFe@6r5tuX3m$Zh<GTuWKG0rtXA;Q-}C3)N5$JAox{%_I2TOZjXCAG`=<
z8T5a6hYH%+N~j9>iu&(7_1}5wzw^|8=f#+IUg|T?GnPFsjhIc)lyY90!E8G(7Uy~T
zAm?cx&(l7h7aL|fZf(eR<9X)L&P&I_dFecTUb+^~i}T!h&hR_W`JLyb*Kl6?s?SS5
zVFqAOe_r^%JoVps8S$N$QQdj!zw?~WcV0ZE^WsJNvd_yHjDvlee*p#4?>w>+-AKmr
zoF~x=X2#Q&p_S4dKtB_@=w~{iBawb+H2qSfBc6U{7X3`Fzwlpce?A=J9EN@LOUaKu
zxbPoruJ>}S58U)iJuu4kF6?>qS@R(`mVJ6iZ7AO%@oK{OE{V5<er_ogC-dD+F_u4q
z9y%0Y*6-&#L)P*RRms6pO<$#gw!LzieS%BODUjby`&UYH7Uyp4<=hRZ$KH_1T*A?n
zv`hSZn>_pnD87Yk@W3G4Kz?5V-++hV`;ZF9;2V4g`N-db6*?giw+G-^I1G8v2446b
zprWAp9L~Uf)XQ0r3l~$zN5G}Pk#d@7WE>oX9H@jJLK-xK9iD`HX%8NRzhV9orr~Ya
zgxl573s3W2eb51K!5gracME}1cmg!Eh1sBm9LP1%{_AM}^|b#`K%1_oZ7)O?8EF5j
zY5xgRf-J>eKpfxX{VSGf{}*ZhPt*S6S3{UuWSxWYAJn^;|AR)%Ci)0XHpc&iG4Esk
zKmGqQ{>Qy-hCcT^eQz7(-%0sLcKIm($p4K0XJ~KcX>X{{2atorKeWexzU05g|5KEI
z!nlzh;_y!VYyO{j#)kss1kYXq-z8uEN&0Um4Xa5zJ&b@0dlBvdm|rH$i<n=<%?V#%
z_Q5{VpdEgR+X>S6QQV$Hew%pT$88<g?Z@pqxFtX#ZnOA(&Nbcm{SR?}i2WS=0=FQf
zou(;t6U6{fpVj<<{>Bip6MGN1He!c4ipp<8_~sx9_cn+Jru8+Euo0pm24W!&{>(Lh
zg-PPnB$NNU$$!YfoSV%0-$v^Ho#cO3fV@GNmO$PQX1L}x>{H<5Su&9;crU6yO+J*v
zYU~ZjE08wiW@Ha?8`6c0NB)Vv&K}aC4jSMF-uYoL!*TckK7zkNC~2`4eho)pn?Jpv
z7Mh`#@2HROsUPY856mEP2>UQ{B$fH!ea!!)GtZ0k{|nrU{4>|x&U2)JCfFs}v*@Nn
z4&*{R?=}GW3+T2_qkq5DCxwfB=(zXM|LqgQqQCt|$9);yc>K$u0{=$htI}7f>T~FS
z5T+LD@H0;JOZ`m0G|+EobkY7p6Xxb@^gk?&|Dk1LzqHz@hiMaSj4SO)12RQ??X;DS
zYU*7R^)B*$zMYrhHq!0ed^f$z)U!J3UBdkw`=F6}6FE%S5!^<HsAqlDyGRdqFYZ3#
z8bgj_*JPlhUu{-eqgmxZuFkA<KC{Z}Gpl^e1unDFmz!0g$*hX-FUIY{{2z5i=}z81
zf%m6>Q4z-;0jP>a|7D8z=e=uZcz@E<2=%Aw|B#-Iq=|`iYa)%BF`Gl^|3C}<p4I^L
zUn1%MV7BAdM*pOpI33WzJ^cSM=!!xA1-hfre<5t|Uiv@KpHBZLjs8yt{huuQKl|zb
zU>@B?{|DSWt0#&6Pb&Q%%wxFu^<Ol7j1NJ}xHZSc_|V4qkg;nXZAU)l0we7V6jsyj
zlrv{+VD6aqr-Xh+DQ#WZ5bX@tRuD%e<|@X!)lkE=wZu_pV(vQTU+dqjnKve~{=LBZ
zH|yKYv#ftJCvS;o{X2p6Z~Ux;v4K67^>40gXa3$1!TL9{GmLX_LfPAx$zH$=_Qr;v
zmEN?o(uaFL@eB}dka>OT1sRTIE+U%oeFSq4vwZ)j`Tpnm{+9ye6yMbhj4$S?HrB!3
z!Tdh_iSjuO1C;eyK8-Bulz&7~CQs_r@wavApO~{_D8rj+TfhKb(#8j4yz{uxi!N}m
zxWKKazXgM&!O&En4A1n*2w{5r`lXMw>px0Afplvoju+u~*HI_IIViYRG&=5+1~K@>
z!t_tfivJL3rXS!NMRN3zrV#SsW9q>(Jlki;x_zW~SigLQd3u{!RV9?G>SXd`C;5>|
zex;FLP?17@fo9n&*;9>5yU!~*N4=6;?iHQMD|t4r<Qu$F;H*^oc`xf<WvXb%E5%`D
z%24f<lGt7;J=&<s7Q9k^<Rz&{DpQr`OsZ<iE7gRrne|F-a+#{zSEh_vWvbrgrTnjA
z{EPm-Z<YLw=WS-f)9gh5KbrHuI0LMeGr+8&tLXosTOE9oHB@x7jf{VxBZl>F=tA$>
z3Eh}`pf?5mf77J&S5MOaeO(6oCS?eQr%%cVj4qty{F{^FUOFir@M8ACnCqmBgJ##1
zWP>(kN^&3<vo7m|<Yk<Y{PYu2u;+y6Q%^`?#*`E}r=)mjN({a!DM6MdoshC4CwTu8
zQUR4v1=UakwNMAfi4#(P^n^6boRG%ZDZYQsznEqX7tB+CX8*&qv`kM+E7HmyFx$km
z*pY1!)6%|oS{%@EY?|{grrG~6&HjgJ&e@!n9>VvAOiN$*G~fTU3^?Cp{6Ee6zsdUl
zUu0xvMn>mn#I-mh?&TTU{~7V}^FN<$27R>|Im<VqJ@=;Mz~9M_cR!%t7Iunp&neo0
zH$?}9F{h*mdeAAgUH7JxU@nDXFhIrLQ&I^FT>A;s>^mj3>8GSB;}m)@r=&jh6#u<E
zC4a;3EE~l7w*6Jg&r%O?ZT|_zJqfIx?`~4&Xmsr{+xM`3&89Icc3Ud{9$?M8Gk|g6
zUe?lg)9;}XZDY*Qh242n$rr?%e>_mmKE%0t54ffLRFE`$_Gc--0==7C|173wf@Ca`
zdA}dg7iRx^F&oAVT|u(Ub-u+y)xL3D9I@ll0iB8C)UV^>+%qoSyT_Sd9v5@OO0h5*
z+CsS2xB;;e#uhy;_Rs-oLyj&Ch->1HoEI`69#@EX>1+EqRCg?CHS4IH&z3Mw`{YAE
zbDZ^=Kcb&G&iu@{44)og+%V4g(#QDHCxs{^7e)J|7=>$tkMs4S$D~xhTGRq{PH2c|
zZT|TH&LzuUDLTkAaz3SVwbbqPi4p44Ls(xnsYVtCOrc)RDqw%j{tzh(ACvO1W>pd2
ztSUpBRaMBCR71^vpVWeOG)QV`cU>=%hL57lbRW7*Z1Bn@jd~L~>yrDg0IDhpL!QOo
z9mbe%GwJv)<NhvmonUm7ekS=@!=^FL1(wVFVV@;F^1O!4WIbGe6>b|K25yEOp!<S<
zLn8Av^tql_l7jiC@N1|6PXcG6Bbz}F<K(43KYie(p09pBQmS73hSa{qnbYXZm2Y59
z{v>_zQ>+~eYi(~|DvjvCHPfkTLhs#pHbUw?yGk0qxLoRGJ>w(#;h^F9x`=;b6@7};
zj15AWYl5fX0H8LZIS7YA3*}G?^<aVxl%GE!Ux%;YpRk5?CZ^mpOv!1k!2f0>({P$M
z=%1`*j`>aGsN5*7;;rH?xJlf!J9&+l%kgKahwEZR_xR;9g4@ebAm}07xL!^Np({hR
z{a3h>=KD@~1fGE+*v_lm0e8Z8;cmDOtn~dJK(=JC&PX57O#j6K-u-Uzl@H4pa@;m7
zqvuA&mEjh*@fGnP2lu*VDBUf?nQr<MZt0D1)84xIp4~DK?B*Y^ZgK7(ro6bNhj8ta
zqpVlBS+5ypy}~Wl{WZ$Aw?^5KZ9X^qeHxVIoLgFsxux}}oB5!6Wr}rkF2F0yht#M#
z;x!J{sQLuAH0XypPsJ@?Y~))=b2Fb*qsn-W@_0A>EjRO9ZuGHU5ySkjltk34Qj1#(
zK%X=$g~+0eVaW{`WjyDWytG%C&vMh>aZ5JOp>?<=$A&$d{%@#9wBaL?6W}4dM|2xU
zBoCRN=#hdDkLWQMMtGQS^GNZ2j~KE%Qj+G8(tRE&v$&)@+9MUm{v?&is*N7%{}HJ{
z)-H}n-NK*5xO0T@UxR8Oo<^4k{SOcQKaVsY@rXIhL;tTzwe0dptD#m|xu0#QR@oCg
z?9m)y{ii`Wc)pI<KQaERRb8R=${FWD|Jy^E@<{Jwt?D~ctNM`xkscY`?UJDxj|?9j
zk&#Ic`RQT(cSPJs&xD8dUyt~3A3Nq@4?P<qjaJEyE>K#BmG!T7`61t}&S#Z86YXDk
zfhsWRl-^=x{o5)<)mACqSfC7X1*!yDdfH0+XP5G1t5oc1m&zooR6+ILcBw(uW?7|f
zmsO0I>r<@Ku*WKmaaJ)UTBRw*D$UVWG4HjDW!@?+At$62X`Qu-EudZO$hP=)X^%O<
z{8u~rZ@H@Th*i2Wt^A+CD&54>6JTZkyH)yj6{-FZs|-vOs6p;Gv{;~q2{&@Ao&I00
za`Aj_+H%i)f$}EnlyAS4{;yTWcUmc+dX*iZS6XDwjE(+Jo9H5K!Z8<`{4g8!vrY72
z4(9)C@+fYPg8}!F5Sx?++n8^)p-*j-iU^xjhTE9`Y?B`mre=B){U4ju5!QIlA@#_H
zqYi0Ynv|d5-{jD#X57rXZ0M8Qq=k4|<7{F@+9C>-J;jDDiC(oQ*m!@v>fnBzE{AmO
z(kUm;*PS>iJwtlco0Y5jl5EnSX+!_VhW?*k4NdCRFmi<F8r|C_uG2R8IrsF;+QbXK
zBMuou{sOm=bh`{s<*A{V4jI(FDg&+##u9evGjvEVa(rW<8k@J%XV5E-{m^&}c6kQr
za@l2c(Js$owrAMUS+L7N%o=D(wo{I%v!Ex$F5UC5N-^dxgOxs<T{@!ejA`to9d&4e
zU25X(QXOiitmmsrW1*@DdsWK){`>M&v(t{QWC!z#tWm`B4e`CyI}4T3SE%Zc`B`?!
z<9@nb+%woNIX=5+gY#7OQU`<VSEY2ZLrQkq#c-@p74w`$=L%IJ_tR(SRYAI)#;8rQ
zH@1m3p^bUBHp$)HCUx8^Z&w@tPoVFAw_Z&Y=vATPH7VNJCdEl@Vz6|gXVfO8)t%^n
zz9u_2G5=%a*@>sh*v9y;O=>3Fq;{6L&b6`rU!dw8ZPJilq#766SpO+fP19}Y@O6qg
zGhbO^+t^R=nzZVQlr=U_*}~ey&U3Xzx3RaoNI5J;ssq`X)y6!eO`N_q>9%!BPhT75
zzm2umPU%MuoNklBr8XHl*CxZ<XJo!jMlIyOvrXKtHt`I#vHykquWn=g=QSBeYRb*%
z4wWkHF*EyLidC-OEV}q&mFF@`en=PNe~r=`%=AxA(myFy#fEHU=rc<R;YxL8DcfDF
z%9Dy!1+vm)ma54XsZKRB-sqCr-DauVZx-Y3F6JH0(vWPH#uT%daBtdWmgb#iF&{CD
zWx`DV*DS45X4;Duu`QZe|DykoY@amqt#nBTvNO1g{{KmFR(DBvnnv~D)*E7$KH}+5
zGRwezvkW4KQcBfugqi+tu^P=Zi|c5ya&te=QkQr`G|I>Gjb+jQUo2J{pGIY8TSV)!
zu>NP^yEQ0Xm_g+s^Cv8{C#|B7wMb#Sll{*Y>OYGZ0-REUERD1<p0Y^!UW-&@TBI_=
zB2{4)sm8q~#KQWlMe4F!#fYq5n34u$V}M0WvrcJ3HcvSj|4-4sbV^G~wrZWVh&9C`
zws4EsiKlI&h4H^d95E%TgZp$k4XP`~!uuOkcZ5ZH98UU|*{UzyBK^rzGH}$O2Gcca
zXt#y)S1mHK&qDogP_9{naw9#77V++H72mQ&#^#+ee$K+2L5uV=Zt6q!o+iy)-7=is
zB14958AN)LTg1JnRj!b3*6&|u&ZI@gd@bU0wTO4IMOyK1N$D0#QVVUnS=z`yJ8ri0
zZm}XeQ(DknXb}hI_MI(^*;<&pXkjkBh5t;oNM&`mRA4W6l&G@s7O6FssG6|XIYX@l
zU3#r*h-i^|+>Nm<)TKG9d2fp}IZKquSE3q`IceR@U9?DcYB!zK7RiUaV6D<EbxZD^
z*QF@6MGDhfM9=#baG%n1C8{L8MGP4RRgAkPtW~n(dPIxNiEWkKgjUhTwMt%0tK>(w
zO2J<8KfM+G|5o}Ftx`Nm{_kl;|N9LoMV1k^e6olBNUK!N_i+B{8&aLoBQ^RQRf}7l
zgZ!UwmHOFMX_#zf{MRa`rDE0OYGq!&RGA}M$^TN-!u?u}J?MYrC|hDH``_M>w(3&V
z9;sE1jjhr_xX$oabOuY6v#(TjBYSwR-Xsh239ZsU)r0<bEA@Y?49)g1{%e(y<yILz
z){6djtGK6I(f?`{Zy)*ZYh}KomHu9<{EYT~l71t@EX_;IdHfoA2~vlwLKY$Gk)y~g
z<jcsTNJT&P`^fdk2ay|)W%S8siR+Th=xRlyYjP9w#5b~L5X&6=9&|}!L~i~jbIIre
z{(c*4xHn_Q{|8&?hTztW+iB#L@ytmR&dGK2`2Un}f%t8OHQSlvNB)CreswGD&BkuY
zwsnbi*D3mQU6O0*5*>9w9x^|}ECp$&M1Pd}YZ?8WxZg8>iT(}~za>oKmL3!8e~V-%
zuNUn`=6`2LRc=Cu=pZl5r}B5NXMUgg|EW<`I6?W3^{C>#>%}l_P$lX7|7PEMDa&O2
z-!-Z#cKKB0PLHYzT`$#4^43H!|L+)8b*UX<4B#I<#NQCSp8l>+nRKJ7Dbu5x^`pw1
z?NgS_QPsjBSgqCTDgWJK)2|o1k@<fn*V~!TclbQ2!-f7Y?p?<9=#H+JZp=M!VgCnZ
zvwwm4-zkq8obafj)9Yop+N(xp*Hizm7Z<pXSj1yn|F!;K#HbpZ>_GnycA^UizWB?<
zn>4OGF_*Jfa8$VhM%5_uaU-XlGK?IGyqvj(%Vhxip$~eQqw7f-Ro%(V$)#P6ZW42J
z$(J)%>l8=Q<<g$KO1jCTN%GK^b-7rf)zZyek5*Z-w91TZPUx1V_-@9m-SiDhRfAEh
z>cid?V_dh??JH5WLoHGh+>K79R#g$MvQMijGP)T9WUI2+ZuZP|OG$D!dosGEm~chD
zZYc~dQTipy`<^!?e;4(Eshc^S7Rg2C7^yEZ-jr;lCbCP8x96&f`|{NB4KAch9lx_g
z-NpBIyiUh|GIBZ3K1WSFkfV;9k;TYu8vfUz;oly}T#c%~g`p$w7st1%ryr0EkHG&x
zC1ijJ_QGK}ewO>>a-RoA)kK#^X(l-Tk#A8uBa(y6#jaaK|BUZ7A6bC?5x(^!++zxz
z$qASN7h~0pl$El5jK$MgCn~sJ;;|>fPS^#>&?4J7WBnE}f51N5kM9us+1sS;v+Kkv
zw~Fm63K~j4+4N7esbP|{Crol<f<(9AQ+a$N`Qc-#AS+DtJHya_43eU0pDGUR5d+th
z>}UO-@MXJ3Rr!#Q?|)2Hx_qi?_o%9N_*BiV)l$oKb&gIkW`?o;&%8`>7~g-8n9|2o
z6ZU45kIarRu|SI}Oj^OJ4-#8Sr`TOV(xw|z?Y=SPFoj9Syi0YKhe_ADFxG#=q`RE{
z|6-W*E(A$mj8FAb=MBK%^r#v#hSC2Cl95=S8Vzno-(pO;iQ5xE|KAqI{7e}8UxH*j
zz^60;F8+TpuCy%3<fN~q{};@>msjOQcT0ZeTIvtxzmqwCbE8WY&3ILD+_*9v@v4#p
zmnub;p`%fb;#Ng=hg9xfD^*-yjWSFPvUcaVs@u<eY|>h(hX!Z_6Ey8*?zgW)%*+p4
zOyjDhdR(;*trhFcsIocMihX)5`af%>T~Gi2^jhi2M!zh{tGf7ZozUIqQa$I^O7A)J
z)8f6VKi0wcXIu?tua%($`u~QtGNM~6qtnbsM|hQc!KFO&<I3w?OaE^z^PkLDhj^7M
z*2x|Lr+9X*6L+v%`DO{j+QK;3jM0bgi(JPXyGISAuVY-WPKGkqQ4iQ<WNKXbbu1hi
z>)4~VPC9iC+IiL{cJYj?JM^GT(9QE&lGd@t$=b$#uWCWs_N)^tvJGhmV+LuY^Qs19
zy}_+a`gQ0?FgL1O#~xval<yf=W%I19Osqr4#3|L&>!fO#wUu4ts%FTo^u~4QsB$)H
z{J1Jy<ou;+@@UGfN?rV;EPI_~J6Y^H?N&Jv<navWFP-zM+^`PDwuLIs@0Zvm*?TV)
zZ4&G5`!1E-2#?b7z2rqu|1a?UZ}cer^q49{38IJva6=4baDnz|qJ#FMNR^q2RJpB4
zRgjmJP&Mt~868ry<d9mZtET=hFH-eGMXJF-|6kuBCTL<z*bL^44zWN>R)@4^(*I8`
zQnrjD^glbK4ccQI^#4fP5C{9e8UO9<5NA@6>fTqRdQyv2FKtyH^e2)3Fu30#LogiM
zAtNzG)Zs<S6<nm;yt4<qnGW&6Sc*f&K})5TPCc<73^2yK<r_Dl)6AR~_Ls01xi*Uf
zvwmr_lvHnGUUHN4`NQisqxZH+!Z5p_e0Gzxe#DsW=q9N;w@GGkGZIfj`6gNNn`bu3
zGO{@&Rzj%HOol1O{hP!@m?Z3$#Z9sk*%q;h`QOcwj=AIXX30dhM{W{F=4R;*-y}U;
z(-pE=oY0@RiFq;R?BaMPD%s&Zn`C6?CemRO`@lDeJ7p8^&HE6>g?&sPE7h32Y<m0)
z@1}7@OLkV2XtSee8=@rF87;c$XvuR$Nq%{h6jVoX##gk|&?Xf1MN4sZv>2eo6eXox
zQ%1P*g(#^QijvC7D5+YCV*D2+H8W9CI}s&y*p1lh@oO+fiGzFh!7TUoA*sIEI}*kC
zKT297qU0!Uwv=eGCqzkGSd_GfbItx}nZUg>E?T+*qS1efmhQ+X=0Bq7|3}exkCOiQ
zC>hutC4-sl|J)NL!>Lg+k{l(Y*j?D&_<16tWYK?bI1T4OW7tanX{%_R?0@xbW&Lj}
z>wjBW|Jf?}Gh3x#eyiwDZ{_@_t+GV;0NS}=2!(JcE9c*WPyv-t)wflup=M|+_1`U=
zO>>JF_uRty54T7|#4XZjyoK-o7RvTEY2LMs@!vMF9NWhDk9LX;W7dgnV&fot`}{WM
zf3`{cP@Fhianf-*PCEPQrE9WAoPD=T_t5PU!To!as_6gL(EqKc|64=<w}$?2J^kMr
z`oA^we{1Of*3kd0q5oS$|F?$zZw>w58v4IAa@3#xSsSD!dxOLfpVhfVVvz~ZR=z<}
zk&bg)`2WHd?!QI4vbTs6x=kCThiiIW8>DYxgY*w=kiCQ*T-v~T-v-wEHps}t2HA<5
z3wtusW8A=617!^|AQLnRTO@mVgJ=UbNKV8CNykl>vPJR|Hn0w}K?=e*py#uN^xs1N
zbqn(-TNr<CLH}%nlyXfO;mXrCNJabx+Mf+lmAOHx_iRA_Y6I(U8>9}q5qmv;4G|lt
zTW=Cm@az14;U;MYjdO=&5A9$KwS)05=U-sgxn4s5o%1gyIR9di^DmY;|HAkZb0?fv
z5b!PT|1Bvoyd<RwLA3ut{Qrh^fE4uq_5`7a6oekq9a2L(Uz-smb+{Wh2GRee{f|E>
zjY!k<d+dLv{f`V1^FokV7K5aPd$l61;k5rtC&j+;CC0xmG5$YEngvM*_v%c3;cNZB
zjF<TSgQSOh^v2Ns$DyB{7$gIP86@n`GWu^0^xxn=>#>=v$Fc_M4G0z=jKMhge{Pr^
zkKR?{*LqhM^spT0Ww9obzi*23Ptbp0{k;&1SdTAWIwpop^dC~tf1tf9OFKqil=c5a
z+9>3O_5Y)+|EK;x^nW>bq$ve`OfWmpH-#1#`hQ@>Yy<l=`hU>AfW9eoETR9ijQ$Tg
zK2CIfx{*EUzmr~c0Q=xS|IfxeH1VDcqw6zbL;vsCd#p#J|C5IP4|p@t|A8^g<De1x
zpL}y!Pz~=bP(LAuzz37?8GMBMSr9&%AXo`s;l37WiJ{JdD==?>MZ(>Iyc6z&FR?#_
zd=$P(U)Dj`Utng*Lz4?ZA5xDXLy(QgIxs^o41$yJFCve_2{;LTxUC`l5c2QX{{jDm
zDcYYaaDOwN@gnzf;r}h{p}6nB{yp&F_Cw_VK*z?<`0p?0e%LtoW8!`3-gy?iO3qo?
zb(T4b&!lDFJJL$oFuOSS1MJnD17hM_5W?G#?vOj=e~D)l>4M*2&WEGi;|TQ6aLz|C
z=hP&hkv_`QaMoGoSl*=%#W^);cSt19R!aC*XoC^>Bm55j0NSB{$e)m!Wy&dKaSrqQ
z@K0F5IJn-&xiZTWQinA19!)8n0|Jd;f{JX;rAb?0ocgYm_i-)__S)>vSf_Yjs*OA3
za^koFZsa+(Ag_X3;9Ky0h{OF6<UM>R_agrbj`3dqjr<w>FZ>uLaLY&DM)(2bB22<G
z?o%)cCfa@rEa5&2-Q-c?_2f+)V;P8m?{obOVdF8!z=M#2+Y`t{_$fREPs6X^5Ili@
z9&(=RDv&?MUW5Gq*t!!SO_RIM?|+D}!w><4vAqr)-WY+s7+4`RJ%=>1fz^p5G+F`D
z2(Y#GAPF(j%;*AfgyrbE@B6;*>Z7{4yY9PcnW|EE&q9rc4y2hLBhO6NnThORW7!Dp
z25Wzww_2<>?5Ovb`5gIV<~N_`ec$KF{N-C`*2l%yX`6n}_W%1#Ux)ZR*1sZN{UQD+
zTKBxZL7SGhX!{BNc)I!L`|hv*jK7q>o)+u)>uK4(6}o(8eEtr9{;U2!#g+ZIw$Gpc
z=l(y1_{R5)?;G$N`$hlXLOe0IACrIcqcKI(bcuSO@%xC>|HpnGkp?Xf(ePin2bNFk
z_foWd+3zcU-R~>@f!|l;pXjo`oA05AY8_KY^*{7u(Ljwq@ng|M%|G&I(efid=1b<s
zqKwKv<bU*0<!}91R8h6%8mj*_ziCgk-}{DfGrw!`Wxs3jUHv4Mo9O>Df0pm2=Ih3X
z`E07h{l?%2{@m|ev>E5Bx);ONN30M2C7+MD{YU&R2JcR*xXm%$;-0_tI~n3G>sR$H
z%)Fn^M||ZmzMfC%Q?PzbJo^d0pig}xCVqfV$i6YhOk2LR|4K~q{7ljM6MjRTR%wm4
zUv=K7G|y2yFW&{_+!W8rWBDiaY0%<7)TeRwLL}G0zoa+m-_eh{&-(}TQ}omHhpv$y
z7XK2xhra8Y`gQUDYWw5juTYugYWkYxdMc;Ck*--hO#L)XyRN-uaf7rkh`Yq!68~%J
zxBSEU5`L6MEKkzUS$-G&t-O9o{5|ng;@_s<rLWN6*;X&UBK@@Zy7)h%7c4J`SG{jn
z#arZOE-wB@bj$J!^k;PO2fh<!f8+a)DyZ_ma9*`dwYWz6*8ZdK@V|T~>a}eHHS%ON
zQL{E|`TC2|`s8=?e>>0A@!da-&aXSKpLL#p@}Eb~Gp>u@;=_3Oh3Nl;>*VKM7tX<u
zb20p|>x4#Mc3x?mCTQ|+oL8ErOLX~V*9Bdr8M^kc>w>P+4Z8Vj&MVE+0xf>od8K7q
zq19h=UC{dX{`c5Whs`I=YgdPDb=WE5?|Rnt+_)*;@|?MThfj`H_y0w#iI;6(v42s#
zWdD2<pPluaFaH%^2Or+wc-EzB_RZ@2Ed5TE=hMsko{h!R(q0lz+CL>8w>}{r{T|=n
z&|k%{c!UONi27-Oda3x|38i@y;?6JsRdk8l9or$VR&krWn#C<u?n}glFSA}98h+rf
zqE?-X-zKj9Q@=G#m6ofhyzQ2LPMx%8e2@G8y!8G5vg_an_0#>w^yBoO{+xSzdd>Eq
zw*C{AU$=ZjiMnn3hqjH2r|JK)?O)Nh<)5_ON7lu^u>39hN7nzH^?zykXD$Dr&-{O5
z`G>{-iSOi}692sIZxcUGAE1Y=|G0hsB|T>OzZU;9+kaC0dGXiir!D_uYPbA$aliPV
zB=_yo&s$!mcUxW;|L5)t9+T!>^nPhROy9BnU+ZJO_~-n;f64#*SNea&mDa2NmjC<j
z_<zN<*6VDqx4+?Ue<&L7@PAXY-yUf3f9hM`|B&(ThoW8FVXknexXXICxaaE+Men<B
z>Hm31|Ib7Ee;(5R^N{`@|D&Bop8h{#bnBrQ+keRY??e0$59$ATD5k{I)-Q=K|J_4z
zMSRuzjQE=Uv*PR4Z-{R?e(wGLzx|UB`JD#;-~R9J#L{m)<o_Q%<o_Q%<oWNRSfh2z
z8~glUpLi&?KJ`#+i+3FJ!;d@f^rQ3>p6`BK{152AqMxE))@DB~ei!``{T#iAo}$O;
z!}L-5b$Xw)AEgf!()M4m?Q8VE(eKz+F8&SillJ`@{ifwl(I+gA+jo_&(>g8F3$#t&
zpcm;c=(nW3C;kq7%JTmy{+vAio%jcx&!=tsQE`{g#b<5*GJV#zXXstFf6aQkxSOui
z#U<x|%lQ{qIM<cm{Z~=-vhz<hRQos1Kh;wMHNNcpQ!}+t>(_UEukA+rr@j{*^68x7
z|DWam7x#Du=>4quR?nGlB_8;?`G4Xe>%-!aCwBG!x&NnenxILVqG`HBm)~&zPk%rE
z&+;r?ryKPDd;dRXJ{qmi>eGB#v`!neNn5l%wHrHQyRlxhfj0SzGwp2o1v;Q_TfZl+
zvag<g+PKRFdKa}=|0VJ7(pTtf^gNYQHFetGE`Cm%_lx_if7$Y1Sf2D=OyXhNF4G^_
z{>Sv=Pq@#w{&maq;tSSS#2*%y+4kGw-=*j2YgA?1rv1~F=c&$m6D?c*Q`_4te?$B)
z=$q7K{a@SnZ|Dd8&cF}TJ=<Hox7)n8+r=H<b^iX=`TNs<8>Ro>{=I)2y{?Hq`}(CR
z{r~nm|2B%>Z+gG{T70)&bdOO+<!7!Rs{EeM5306YL$&tRQT-47X*9Sl8kbx@u9IeQ
zi}g`?jL|qv&?HUKG+m<0WWHrwr5U<Lvvi$q&`p}7c`_$67HNr=X@!dKf0oHur{aIj
zY|<8OQ}MrM;w|x(KP#gOs-!BarW&d(^k)}KVUecx&&!4N)Ic4DN_#s>IdqivcG}um
z+S*y#+F9Cq$=1uX5N|2g|6D1r%H!3>La(+7n+lbdnk_Y#mRc;el$Kg8wU(BOv~926
z6t>4(?w9u!L4W*Y{G`_}$4|w76aQ`eqxk9g?-=HO=HZ{Ew--Ta=|ZveND(|*1dkQL
z<3;d}B6w#JJRvBOT_}z!k`^h8ghjd{S&^zp^lo|&y_ft>;Lp6DK0v=hAEXb_hv`@8
zBlIMFl>RAwjDC%No&0d{&wQLdK~K>q={M=O=(p)<`W^Z-eTF_uze}H^-=k;f^YjIJ
zmY$>M>5KFw`Z9fmzDmDOe?VWOKcqjRKc+vSH!3fd!tzpBQ3|U{VRb32DTQ^Vu%Q$-
zmcr&z*is5xOJQ3nY!@D`7I_}77P%d+7RMg0>FoWP!}a3W!}ZQ0+?XiB&9YKh9D7tX
zQwnE`uzs-!n~J?hO-1^n)}|tC+bqI?fg&9A4|RU#Xm(n7+%r{#gSDlwDDimkauJSR
zEy9~cZYO0${wH;7ML1BDelodTgi}S`PNs@-Piu<wr!@;jI6qc|3q=c@)f6>MHw&&@
z-dltXlSMdG)F)pm%FQ>6dj56!wIb}QEW+N&zaD+C@ZEvZ!f?@6e?3tg_18Cx8vgZK
zQMd0_6ghvlv1p&~cFF%|<LBZZ^N9Yuj+eI;ueZf7#M|TTZ;iLVE#CfSS>+?~_LK6p
zcsL$@Ydrk6c=+vJMWiK)aIjWbN+3nC|KZY(hf6yiF73Dw4_~n3LTSf^(vAzI9goDr
zkJ#}@X~!d_O^?RIkJ|KTY15;nO^?OHkJ<ECY13n+O^?UJkK6QkY18ASP49?@-(k}`
zN}JwM+VswN_?<Srv$W}*rA<%7!%x`sL}}9#rA-jyVL<^VQrQx85vq((`9ehsl_Hd}
zP+p~MN<c+c7vjQO1L=jgdlgZ!`$DO4E<79;N~tfDn&g6{rRKO$YKRMu#D!8LTqrfd
zg-88jSLxVN16+74E|lOeJRTQH$UF`gN}#?lFO=|nn_VbDT~IN(2_03+qo`h~lt<!`
zw}$rC`|?PsYL7e|kCZC)$c1>s-bdq+$KsL4<B@m7Bkznyo`^?eV`C|^BDAr{@{uCT
zQcWL?N8cLG*Q0NbM<0$yFT|sd#G^Jo7LPt2kG>-weP=xSL_Au`phWa&acGI^v3TsQ
z;k@dCf6V^JOW`}C1o~K!s&u9wkH_B{PIU?U@nUo7Y`-Jk@z!v<-%+YmsVUx3>@1!4
zcg8#48cuwvVcuD6E}i=);)%D0lV574CyLFb^Z%}R*Smf({`>eJ;uphDK>s5BQv7oK
z@=wMu|8o43cz3+}6Y=ghYrCF{_r!a?9Pf?yzAxVU#ka+KZTrIK;(hVH56Am{JKpyv
z@xJ{n+K%_f2jT<o{>}K6_?7p>uYB-N;#WTQbo|O6#|PtsABzt@6(9Uu*!ZFNQ1PM<
z$A>-<A9^Z2^qcYF_|^E;_r$LXpNx;hM?M@M`S|<dBcG3tys#2a#*^=fC*K=SzAv7X
z!IN@*@~L?8lkw!imH23U^uzJd&&Nl<82>ar79aayeC%US(o^xVUyqM{{QV*Iuf?y&
zuYWauBYxw3@f%OZZ&>(Rd^|q>{*T1RW%%(Y;}h{zJoO>3k_092PsS(T8=w5`_~h@z
zZ^mza$m>J#Tk+fRKgRzQpNfAL|8x8=@pL@>w&&g!PyfQF>8tVdi`(%#@jFj{B|aUW
z{)I2nAH=7hel|XB^)vD5=e#b^Bk`H|%zNT9PseATjn6#ib%AXCllY95{A_&osrc+y
zJ{P|mzgxVXp=YCrgwMt2;&Zb6z4$$BzxQ-J6VDVc`g}a|d_42Dc;=7d^YMlF!n@-O
z_`V?77oLhQd@{aJRK>~rg77)73-m~Q;rHVUTJqU=_Ve-VbMfr+@$45r7tj7tJgf1a
z3#lY}?%na+Q}Ns<<GJ67=ROzDJ@afl=Vf=Xzew;!t6z=hwA_R6LOdVOzdN4)Ks^6c
zJpajf{`2wtbMgFF<BRddr{YWTrJ~`!^zQi52jWZCKN(;8bbRTn@#XkReC6rS6e{KQ
zAifY^jjw(szWV#|`|$_ywfNdIUL`#fe;9uhfAoykGx5jqC*hw;$JgV9@pxe(UYLp(
zrsIWc@xsmb(gy<5?#0+Gi{0|rt%%*q*sY4)>e#J`-P+i#i{1LzZHV2**lmj4=GbkC
z-L}|mkKK;g?Tp>7*zJzpzS!-L-GSI0jNPHw9gW?w*d34EiP)Wt-Kp5U6uXyW_e$(u
zjoq2py%xK(v3ossZ^Z7+*qw{r`Pf~E-No2lirwYdU5nlI*xiWT&Dh<F-JRIG7<*;0
zR~~y6u~!*;Rk2qcdo{6F8+&!JS08%~vDX-TO|jP;do8io9(x_J*BN_VvDY1YJ+apt
zdwsFjAA19_HyC?Eu{RuhBe6Fcdt<RT9(xn9HyL|Vu{Rximtya7>|KeytFbo|d)H!b
zHukQ^-i_G18GCcFw-|d%v9}z1E3vm0d+V{c5qq1lw-tNav0oAUm9bwH`_-{u6Z^HX
z-w^wavELN?&9UDS`>nCx7W*Bs-x>Q|v2TE|C-!?|zc2RtV}Bs_2V;LI_J?DCB=$#R
ze=PRLV}By{Cu4sq_Akf&mDs-;`!lhBE%s+)|9b4-i2a+fKNtJ+vA+=ei?P2H`^&+h
zzrPy$Yq7r`ocQ~j;S<W+>Ko1Rjn??aXnbQVzA+QuxDnsD8Mm6^R&(5HiCe94t1WJ|
z$E}XI)fu<C;#POu>WN#uajP$G^~bG&xHTBJhU3;q+!~ErV{vOdZcW6k>9}<%Ze5OB
zSK`*yxHS{EuEnj{xOF{l-H2O?Sd3dsacebht;MbNxU~_tHsjV-+}e&?J8}DB+%AjT
z<#D?rZdb<bs<>Spw`<~dZQQPl+x2m~A#OLu?WVZh9JgEIc5B>ji`(sSyCZIQ#_g`S
z-5s}k;&yM`?u*;~aeE+c56A71xIG@Xr{ngexP3WpUy0jS<MvG4z81G<<M#EqeIst)
zjN5Z@dp>S2#O;;1(;9c$;!b<q>5MyFai=ft48)zmxHA%W#^TOI+?k3y)4}|I=StkU
z8h2*m&TQPd5qECJow>L(A9ohw&SKnIiaV=uXDv*Hc(E*AERPo};>Egnu|8gGh!-2<
z#nyPSEnaMo7rWxcp?Gm5UR;Y8H=hrWH=b=g8VNe$pgIm}<Df1M>f@j>4w~bjB@SBS
zpe+vC<3NJWIOvLl?l|a)gWfpki-Z0+7>I+xI2ek9;W!wHgV8t`i-U<cn2dv|IGB!u
zOL1^HeBvEkjf0suxE2Sqad15jZp6XOIGBrr`8ZgJgT**lii71iSc!wxI9Q8=^*Gpw
zgH5k@(+B7&`Xqgho~7sLt8uUu2itM56E9tim&)R$@_4BtUaE|js^X>Uc&R2{s*RWG
z;-&g{sUco!jF+0?rRI2PFkTvpZ(fXVmc=(K;+r+`&Bpj<Q+%^IzS$SwT#IjR#mn{a
za$~&Q8ZWoS%N_A@PrN)BFAv4b!}0QRn2vI{Ebf-a-PX9<7I)j@Zb#hhj=Mc^w?FO<
z#NENTI}~?^<L*e@9gn*cad$HAPQ~5nxO*w?UX8mmarau>osGNK<L-^Pdo%9N#ohV1
zyAXF5<L*-2U5&eI@t0HIDv!Th{L85#m??tUB3O>U+!lPh)$3a;MX*`~n}Tl-#<$zN
zij}r+?Zmg+y^56%uOjaBD&nqhtrdZ#?jq<dg1&Dp7J;SyA{g*0#YH}YUd75#e0#Wv
zM~h&r2*%^v6Gc3E9!NV~EL|#s%U(r%<w5Z6veNRm%SEN-V(F?^v40-RD`WA>c)T(Z
zuS~`(Q}N2Q*DIHb;93#P7Qyu*m@9&L!K+n8P+bHyMNnG=bwyAwc&*s+TCwA`V#jN(
zMbH&RmX}_wc;#}D&!tx@{Yi7lBfVfTUb*a5lz6!)QBdS{xhPn0zU;WmMfsPD@-G+V
zUoNt^Toiq|NOC1!x$0HqA}CT7k-bG^|J5RE!Fht&^8`w`s)SPF>*t9}<y|f6d9^6-
zYH8=IRi(p3MJA%+z*nnEM~I5-iz;0$s&w^yt8Fu0#c?x5ZZk!0GevGQMbeof=}eJy
zrZ`-xYen9IVp|avTVJhwOA(ZsO!^X*pp^2wcx&e=i=&DqnUoj>rPZQDv_-B(RHQgx
zwuIt(QS|lVs8TbO8llt#H{z9>UV2W7+RweR@M`U=l@>}HUTZ9>6a~_PSCNOH$VF6S
zzi_@>s>wo8j^KQIDf5LQ&xN9Dg3{6FP}a^jf?X)3FDmQ;Su7F>N~=ZQiv`VjRHP`1
zSS&JKENwkMLzd6i<hEGkwpbj#<W=M#co3Dg7C9^xIV@@Kwc<Dn#X+U$Eu~Z7s(p)n
zm9N%Yc(qm)D<33$t-VNCI;d3U()p2!TlOj{zg!e1D0UQ4v8yOkP#m{h6un$pDy1m)
zdwZ@FN39f_1rL@Vtd&v}X;+G+Rj*>lnpaVrptO2kS?lL(rOH_@jw>QZuAMJiTK6iB
zT`$rKz8^_jtQ9GWwCk2$z39CwXf6e>wiQuJDL7wlEtZQ8I8iBRk5|jQWG|R4g6pL~
z8LyT-2#S2mlpwHF?p35Lf3;HUilh{rZz}3mtd$ZLhm;pf6<)7gDgwn+6e%l;Vk%s!
z0!IkG9~H@q11pLH?XUDXD3~aMsUnb1Ws#5IK~x-St#m+<d1dJuEJZRZ){3l(5(TBi
z4}wzS^U4+<e5LEEI5AP|uPRC^-ULNZq%0zNh|YJH3M_F8&eIjwz67FIZk#U^d7U3s
zyrYZd^CZRF?Hn5U`Leo}ma5~`>LRV6C{|P)QbbY|kt3_m_ms-Be7>i|SzVM?{mNVs
z%nM4#%D6c8LH<7HEaR;{-zq;VHAR^<MRLJ|<?}U%))a--_>A$r^7n5rpFE3EWKmb_
zkf6ReOmM!e?)B$wUtiRzzNk}uQK$MMTfz6E;>e;--n~U2!}?MVMeYvNl8#)9SJ%U&
z*H^dW)t$I^G455wy~?;(9rtSDUTxfKjC)OSuQ~3u#l7CRHx~CM<KA@KyAt<i<KB(9
zHy8I7;@*1P+lYJH@!G|Btt?(EkJqZgx1;aI*J>2#>+!X^c&#B`YYg9#uJG47;`Pa{
z*C#hc-RDuyc{F$)jh#p1=g}3(X3iIHoJVuEEXM28W3NxkV|x5Nx>1TQHN@+e8sqiL
z-LGG^^K#F5G<Y7(oJaHL(P}BWHum~8DX)!}qS@MbeO4Z`^0?j^uivOA>o=^=*|%_!
zYN%MYZ^6DL@lp{>v(it4G(^Mk`dZWLtMl>t`poO=edp0wyuQ)&`nrvqj@i2S`u38j
z{Ppb>QR8{kb{=(~M+4{4-0NEo+7f{mZL7+5Yw7Uq_EM7V!SiVBJeod_X3nGezdvUF
z{LqEJ->^_@c;n)9yiqY2`n#!&%Bg}XsfwzphH9yf>ZySmsfn7Yg<7eN+NqPesGo|w
zhiI5aX^h5cf-cb&nxShnOV{ZJ&C_zcQKep0%CAyhRS)%2AIYmqURCm{l2_FvO%a|d
z`Bq&fc~@N}`B%xmO8!;yuabY&O`0R+R4vdVEyWwvxT|qj<E_S1ji(w<wfa{puUdZ9
z@~W0swLGfjQ7w;Zc~r}zS{^m(TjTf|b*|AqH65hxHR@fXEow%{aW#&+rhTtTdriAu
zlm43Y*OYl}gSO(0S^Trk%dB&<=-e!Jle`wS>!Q3CF)vBGtj?>_u1dQq-KzAfSXPVe
z%2-vWRpqQYhpWn3bq-gRx9S|ODs$C2Ty+jtF|R6rRh?IHuR4#b%3M|Es<vOnu<E?7
z4v;#ms>7<bS;e&KoUCf2Rp(^YIazg1R@HCSIazg1RxgutvZ@VNos(7VxT+mjos(7P
zWL0~vY8UUFH&(Uhs`gyfo~y2d)fHN$HCiXktDB^aR<~&<-dMxGhJOwJ8vZr>Yxvjj
zui;<AzlMJe{~G=^{A>8v@UP)t!@q`q4gVVcHT-M%*YL05U&FtKe+~Z{{x$q-_}B2S
z;a|hQhJOwJ8vZr>YxqqLe`5{*8vZr>Yxvjjui;<AzlMJe{~G=^{A>8v@UP)t!@q`q
z4gVVcHT-M%*YNuhp*Pm>ui;<Azm9($|2qD4{OkDF@vq}w$G?t$9sfH1b^Po2*YU68
zU&p_We;xli{&oE8_}B5T<6p<Wj^78`8|ygNajxTB$GMJk9p^gEb)4%s*Kw}nT*tYN
za~<b8&UKvYIM;FdIDBIr=Q_@Hoa;E(ajxTB$GMJk9p^gEb)4%s*Kuy(+`zeka|7oF
z&JCO!I5%)^;M~BufpY`r2F?wf8#p&`Zs6R&xq)*7=LXIVoEtbdaBkq-z`5Z(Z#d5z
z&hv)zyx}}=IL{mSH}G%Z-@w0te*^yp{tf&a_&4xx;NQT%f!{BGzp;UT1OEp84g4GU
zH}G%Z-@w0te*^yp{tf(_I5%-_;@rf!iE|U@CeBTqn>aUdZsPQ{^2R34O`MxJH*s#_
z+{C$wa}(z#&P|+~_%`uv;@iZxiEk6%Caz6fo47V{ZQ|O*wTWvJ*Cwt_T${Kyac$z-
z#I=cQ6W1oLO<bF}HgRp@+QhYqYZKQduFZnWdEIngH=Wl_=XKM0-E>~J@NVJV!n=ic
z3-1=*Eu338w{ULZ+`_qqa|_=VzAb!P__pwE;WMG(jV<SK3*#2PEo@uP+ZMhpd|UXo
z@NMDS!ska4-q^yog>MVr7QQWfTllu{ZQ<L(w}sEQ@f*I1-`K*mg~yMlys?F&_%7X6
z_ic6GR`+dn-&Xf+=W5%z+QzVrVH?9XhHVVn7`C0OZRctm%Qlv6EZbPNv20`6#<GoN
z+d13Dv+bO1W7>AkwsCE{CbzL|W822Ijcps-HnweS+t{|TZDZTUwvBBY+cvgsY}?qj
zv2A19cJ8+EZ98|{7`HLHL3v{v=eF~<jddIA4#pjfI~aHH?cm$Nw}Wj5*AA8)EIU|s
z)PG0)chrAJ{dd%VM_qT+bw^!y)N@BYchqx7J$KY|M?H7cb4ML_)MrP1cGPD_eRkAm
zM}2nGWoI+qyx2+|)JgMR52F8I>3n7B&5KL+7Ex)b{7Sr8?Qmc7Z~B^lvsyOQ{WL(=
z<IS2TYNnQWv$mWnsfOyPksMd+xY|jYrWvmX(SNXXzH<G|TA9`sQE92JEZ(e>Wt}YR
zWLY;#<1|ZB*4?0+G)If1@^!1U7H`&HB*)h~zP^fTNgnmr$no`#uXlWdJR8PHz6}#}
zmDI68x(4YwVRcK>(-3b?%V)Ye-kcfwGo>tz#r=zAR8F;2NA=V|jnqWV)IzP)M(xyD
z=r`{Bd2}(G!}pig8y_qkH4J@!>H7Eg9@lj|*mm62^Zlj4YY&zTSATzL;KAM^_ipNm
z`(<iVrY2=-QKklEYEWihSugd|APv(fjnfoeqAPTh=4gQyX_;1Njn--7ei`7hP1=h4
z<+b<AwL!TOSZxZY&HZvYmustXIhT*$uV}kpv3b8jgB2??_ba*{?7d&HaKED8*7H5m
z%vhSB$wK!l8j2L<>Qk&ptL@9xw0yeI{qid|T2k-w%Z2W1lkyv-!kiiFiY-^3n!2v<
zmtQSZ$`VSkq(lu+UX(ghDx*l@02x=DSJ=wz{fZ)~J=Y4AHWbBLQl$cQw$S~GslQKX
z<-xXD)i32&lv8ZJUoloHxAq+Ne3LpqDDS#t*UwA3e!t)^%C)2(7tdF2*dV?7g3~NT
zLDpr2rpZdtT>rkz2TN$q8_S6;pDWU`JKQgyr-DT*pGT#$1WS4KrR_zoLzWh4sZdd9
zsRE^y(pf1LT>!dL%3UetE2WlraOzfVD9Uo_ly4L&wOes2&zEW~6_mxbv`ssKwb1u#
zS)BHYa;l(8s-kMDDO4)4sP+TG3a7rJuGG2(LPh=kie^y*HIf!8tvBC)(Bu^@wzQO*
zvdFu&PzlKSow4FQe0~A{`|3Wppo`?*TNNFJ&J%Z9?7Z(XcO@3}DKfBLoUijGtz6-K
zT6zn7|2iuw>@6`-DzyOVD(xwfb{D#@rkb?;(s?dDws{*@^cMM+@05-za__bKynVb4
zEBZ>USG1)!Wbr1`yrSZgD(Y;>>2X?$B^QH}R54Jfbd?m5^Iqh7zEsp@uu!RtqK-p_
zO0DK<DQzg7)`G=bqGGt%RdBmJ&sR#h4c{+Xy=ZYuF5w&JORD0M!B><sQt15H2OC^X
zrESIg@ci5qHFLpL_>l2_D&Cl-=<lzA|Mz>W!pBR+1WgtyffcnpUn(;4iBjPcrNSqQ
z&!?gam*_HGp`vKzs`i8PX{Fd+9I3rKWq7`%!JMjMb5WgZ=M@wc?c<$xo^#P01+<na
za<t^_TH#Gu;jLIIdr|B?R8gFf;{15qRBTXDinm9*)@`@F-L?)*)#+*)@ToCi|FAd8
zsNBa8k6E9nA>@;TB%jODT~WCiE#dk4{<XfiKdUXSJDE3KQghxYi};sacAih}d*-<B
zUiiLyibLObhZlQkEe^}7<FLFt4z=o`eu+aj0*AGi;;?=+4jap8F%Fyh;;^NQ#H|f+
z*ftY~ozyi%V`RCfhve706o>uWaX3&DhtrO^G#!UmuG4ZHUY(D_YYTCBqd5*23eD1L
z94;x}v;CoGd+tH1qk2-7=l4U;<cFTI4?SNWdY(S?{Cwz{`Ox$7q37j8&&Y?Kj}JW?
zA9{Y}I*dclh=-mH4?P?5A5te_@+^1gIqlH1*P-XEL(fi!o^Lo3sVxrOLm#@=J#=4t
z=sxz){o~>G2JOU=dxfKmGjZgb^T_w(k&FMxTk^;?c2s$lq|u*qR8>Vy)Jc;xMbhZA
zInrmt@kq0@5=Yh2SIf6rdN*lD)m_w0J*13k$5%_QPv%J9%TcvsYU-$w25Fc^$-Y{7
z)mBgqIlgv?lvk_1wHt9%S53;Pn<x3WqdKY|pb1)_#W-qkY=i9$>e3+Z2IV)%yJ3yw
z(}=B6K8@<xsEkJZjjJU8M*K}>R7uh`$)~A}v_q40P3qgEK20}Cc}?<d+9Jm_YlG(H
zIBHS<mRhPOj4dOiel6OtMLV?IpgD4Ut9rCb+lr~xxoE}PsxGbS+9tm?``Wa*_t{bV
zI4P?`8J(?h)Fpja2jS?Ff44e!WAE1H-SX*HR<|;`rSH+MJ@V_3N3SybF!jl|PkRk$
zg8^+gpv*z%ZP5ClIuBwTZiu6iW|D4H9Y@uBRNIfqZ%liPsn@uC#+}=7Wli*wV<(h7
zah>EbA^n7NHL3hbWlw6CNp+rdyw8H8Np+fZZB1S}o`o@~d8eREL7!61DNR2m$&~Cb
z$^Md}F01Zkhg`A!%5)rEbsBtx99>hDYuj-&t9fQ&%-VNdRj)gh*R9`hYHrBmhGTE2
z-VJ%)(A+nizMHCcQ<L75=S|yg+BP>z*5_P8b57r!W9Br$yr$6OeB>X79WBXoNqI|<
zm+W8KjH6}9%j<EpqMj>bB##xxuc*(8<5!%<74`ISaOACjq|50@m(tPN!f~yv^eP?c
zK|0cVbfnklNUzP29+@LOF-Lk}j`W-y={Y&lJ94CV<Va7*ksgdAy%a|qvegT5q_^Qn
zkHV3jf+J7iN1ncqJXs%kvOe+zedOu+XiI)RA&xxh9(jU2^5lBt3G~QQ=aHw!qaB!@
z43Bm+@6K8rdwMv&I2gxeOL1IYMxAl&w&&Olz_BmCV?EHvRh3l4jnqt|beY!UxZ1kM
zxnrLQ$JIAzK90TnkM*)1*DS}ecmHv%W3|C??Eqb;MY6B1fjVd?j_bQ<E{?s+j~iOZ
z_6Bun*re?^Zd9kntvGIyPm{Dw%5U0;<K_xdpXMuZ+@hW>RirMhm|Dl<xD8*MeQhfw
zpZ0!IUWc@u>f5O<-gU=a-E@ss<JddyxVwk?NM7AHNn7>EyGL2xamPIqG!@6a&Bybq
z(W|<>cJ<C4wQHq56yAr&{mA><NNxJ94^)%f28v}R4Qd@O)8j$uy+@A+wa}neA4E5#
zWrt)tq-H}}bVxD7YA`I_$O38oQOlzvbhXgYX3ODbKO=b@kHH>yI>#L_E+cQ~<H^oA
zo+^*yX;r`EfXkLIE8wbqGvXPAU(+nta9mS~S?On?dIulRI-RpCq#3R&@Ve7@T_vuo
z#B~)jZgYI2i5zo7KHhc5H!a^>jALK=$8(qBcpm8dP8=_2pap3cG~a@<b*>*TI%W~%
zqV$XEzohO<PTP`tE-80OW9e)^UQv#5pW_wVSJXwP__0pxW1YFjI%1F4HMB3$V;z#m
zIw6mBI3DXvJk~LHtRwE&Tjy9O+p*52W1T3+J2SK$Cl?23nAYN?te$4*Caus`ocLlr
z@f3FAhU%okvH_J7U#KTlb#y&Wst4mluf<7CYn-^$Ph9FJwU_BC$*XpWmT7}_;zaku
zNu6|c@~V?o7lg?GaiWXi#9QG+m%@pzgp&sA-U26$^6|xV(zG5Y&6VW%W@R@w&=k$b
zNsIdX@;Pa>-KBrhCQX}qx%^Ms=HjGXecSD8?~RiVj2-rMOvg#5V>)f?9HB{4?@sKU
z^6Xq7ZP2OSo!Y5OeY=#`Wq-G_yW8W$+u@`aOYacLqpzHl*@w4J+x2Uoe#`^P^rd9F
z16?B9y%A0Z%ShdQL7fcY9g@zM)XA_kBhrt^-vxg%rhUh><#;2t#mR(p6P72`ZSpE<
z8zY^?^?%}<=41-%lzRDv71Jt68B^*$r7qLToNgj{PPdS{Os|q-FUi-p(#a)dUDifd
z)ai<CSERe5zJ9g&<f`MZYSS6@oN?@obLV|?a$TAm(%;agH<W)<e!g)|=G1>qJ?FL8
zylo5O1@&6c77IAN4^G^4o-8_cQJohtFFOBA+Hy%7EJ?R4pJm6cIIk<}VZiCcP3eh2
zniDsfCk9<kbo!p?*gerfd!p0yWJ|qtmY(PoJ<$nzqI2`aSHj7*^P)5FRA<xa#VTr~
zc3Oy2H{YkaZBEN>&`sK=oj5J8qXwF#`8YLTd0J^-O)pK*Wx7iC*T|=4Gfr#euLti`
z&)sRQblw-IwIgw=tL;=*+o_wrQ{8K)uKm+`dDpkm5Y3T1>bK&wVTz{Xw9)oP<uoqG
zX_LA(b&&OD<uyy!qJD1BPTioLwhYoGQidC|(^mW2>~HId(+*{K*w!Kc4t4D8j?=Em
zIPGa7ZQ^F?v{(MU^6kalXM0~O$+J(MuI<ylnK<>~e%fz&U_4F-n+gAr?L%0G<uNR;
zk+C=()doJKPu*ahj;Y(2wjERNiE?Tob($C^X(zPBgySZaH|4k~^_tSgx(H9Fwa>J=
zUQ)M9j=wCwtMa<4&9ADbYw>i(z8Ps|)O|)?GmgC`pKIzgtNq<MonF^QH&^0xPM*fu
zPv^Qw+Ij6euMYE==5Z`IZb2Q5!Jis~KV8t?i(;SEr%UR!<l6Are7dY1m$jqM=F=7J
zy<)oos8hEyrv`aW4P=}eyf`%wacThK)a}Zt&h%5A=%+fsPYueS>hwOlI6<p%=JWSV
zH{n^?JZ;8Vc?-2sXPoI-IrI2`=7ILCVk^!JexFqi#F@|GGlR}&9!|~-2%ptdQzJRH
zMjE#qXSMS7Ywl-t@@(j#-Z*QljI$=?wXDTiYge3gt;AXPMQWf~vfevL*W;|uHrK*g
z-xA5IPd@$1=~wT5`SwfOZ+pMI1}bQr>>IdD_7B)UXnhdZP%WvKFZVNF?q@^N4q+Zq
zhR?jSkx^2<&%Lveoj7w1osCwJ{iEtVD(&b-oVi&)^WA<n;TSjRXA|<7xIyYRX}de`
zGuO=7w04=kMAzc%QaNeAOWSdFMVtBlKbw)~jN=W0pIw_Gd0bOZc8#;yezHENUFLe?
z%=h-$g0^3h-?DaHS&uV=(Psut&)nLc8MHiGmzTlGvvqYeSa@b2@XSEqneXE>gKTFT
zj(1yoW?=2iz}cDa=QFpqXPfG3K<vzQa%M2<%;3|Rfu=KqHfIKH&J4hu8F)D}@R9~E
zMH;#h>Do%9+4e9Yj>@Q<DyWjGsG4f1mg=aU8mN()sF_-*mD;GttAjeJi@K?YdZ~~4
zX@CZ4h=yr|)am9Jjnf28(iBaTy577@SLiCu&^4N+>xARxO`4;5TA)Q*qGiJ92`1fK
zqjlP#P1>Su+KJ@+q&fU^_~-D?;h)3reV*p<d#|TC{B!u{@Xz6&!|(nq&EcQJKZkz~
z{~Z1~{5~6!Un@(_dz!;Phu<%tr8)d__~-D?;h)3rdoIo4_jHx!@O#2ap0Ls!e)oTA
z4*wkfIs9|@=kU+r_XL*a@Xz7*B$oVlqU8HB&EcQJKZkz~zt5;Nhu?osNWM$c9R4}{
zbNJ`*&*PuRKaYPN|2+PA{PXzd@w@*_^Z4iS&*PuR@2M}%<DbVrkAEJ&@8~p-e;)rl
z{(1cK_~-G@<DbVrkAEJ&&&)KBe;)rlexILd9{)W4dHnPE=kd?upU3aBHO=Gq`I_eO
z&*PuRKaYPN|2+PA{PXzd@z3L*$3KsM9{)W4dHkL^(meio{O%```|9NBHTld=KC{yT
z{ssIC_!sc|3{O78(*k~8s>Dl`+=nOk;Yp`gTEM@6e*wRHljO5LE#P0k?>;3h;9tPM
zfPVr10{#X33-}lCFW_Imzkq)Mzx$fBfZu0DTEOofC%MN-3-}lCFW_Imud^*J;9tPM
zfPVr10{#X33-}lCFW_Im@BSz);9tPMfPVqM&!Mz{e*wRHr?iNlVJ9u(U&OzNe-Zy8
ze&1bb5&t6oMf{8S7x6FRU&QaeD=p$*#J`At5&t57&nJobCoSUFJCGLf`wmRZK#3VB
zd2UII_!sdn;&<<s7V$6QXB0|{_!sdn;$OtSh<_3PBK}4Ei}>B^rA7RU_!sf(Bu$=$
z(jxvv{EPS(@h{?E#J`At5&t57&r4|${}O)BP02lD(u0wf@Gs$C!oP%n3I7s)9=gOs
zmw4#X68<IpOZfG8q$T`I_?Pf6;a|e9BQ!1HU&6nHe+j>P(zJx%zgSO8_?Pf6;WvPp
zmhih@P3{rW68<IpzAMrae$RBtGhNcNl9uo<;n%~GmhiiuNK5#a@Gs$C!oP%n3BUV{
z<QXt6;a|efe3kUdq$T`I_?Pf6<6p+VjDH#bGJemDX&L`A{$>2j_?Pi7<6p+VjDH#b
zGX7=!%lMb^FXLavzl>jJby~*1jDH#bGX7=!%lQ49`Lv9G8Nd7Uw2Xfl|1$n%{LA>4
z@h{_F#=nez8UHf=W&F$dm+>#-U&g<TUr$qVPnCSPre*xg_?Pi7<6p+F7b-2|ch8lU
z@p~pn%lKFDui#(7zk+`S{|f#U{GJ)o3jP)REBM{3r4{@u_*d|+;9tSNf`0}73jP)R
zEBIINui#(7zk=WMchU=%crO$0W#YX|EBIIN>lsUW#?lJ@75sY0k{+_Of`0}73VzS-
zX$AiZ{uTTy_}w=q_l-$UT3W%sg5NVvTEV}9e+B;ve$V{L{YF~Bzk*+{Thi;6R`Bbr
zPde+9=c1&;J~84YM!dv`ml*L9BVO{nlyu@Jo%o3nFX_lnjChF=FEQdJM!dv`ml*NV
zDt<=1#E6#|@e(6m(&?Y{IV48Bw2Ge*FS++gtN0o55+hz>#7m5Li4iX`;w6Iwi4ia9
z8BC0Li4iX`;w47BWZ)n%;w60;i4iX`;w1wKi4iX`;w47BWKbb_#!Chkl7WT9h?f}g
z5+h#H)0i0Xl7WWAh?hJECPuu(h?f}gl4ruih?f}g5+hz>#7m5Li4iX`;-xkGjChF=
zFEQdJM!dv`ml*L9BVJ;}ON@Al5ic>~B?B3W5ic>~B}TktP$MzoB}Tl&h?f}g5+hz>
z#7m5Li4iX`;w47B#E6&H@H65iM!dv`ml*MqzMI5|ml*L9BVN*Lni%ntF4N?mGBM&M
zM!dv`ml*MqZq&qxmvp5jM!dv`m)v(I{XdBjFBxP>jChF=FBxn}23ry%URuY`h?f}g
z5+hzR0FxN;5+hz>#7m5Li4iX`;w47BWN;=i;w47B<eoM$;w47Bw2prrzrLolj(;7$
z``u(9Co$+H2EDY7-=I!%51e%MCI-F4pqCi*(mH<6N67$KV$e$leiDOT(jA-_^b&(!
z(k+}A^b&(!(mk9Q^pe4{#Gse-ZzcEDi9s(h=p_cd#Gsc993=+5#Gselb0-G9#GscL
z^pgIt#GsdSGbhhwN#9s9$dufdC->!vK`$}rCHLn^7j)7EojkiG2ED|fml*UCgI>}t
zof!0zKDET4ml*UCgI;3LOALC6K`(9KXV6Oquo8n_V$e$rdT9ecgI;3LOALBR_jY2?
zOALC6K`$}rB?i63pqCi*lCJN>pqCi*5`$i1&`X{j(gyww{2TZ^M<mY?X#+pwUNT^o
zHt{p=C0*vp^F?CZON@JoaW8G+XWUD=(i7ueV%$rfKa%H<#JHCXwk5{B<XI%?FHDSk
z$umjX#LvK&82AzcUovQ!82AzcU)sdKiQjWeV&qHu7?VE6#K@Nz`I6_~#K@O6@iX!z
z&%%k3FER2ZM!v+zml*jHBVS_VON@MpkuNdwB}TraPbV?*CC|@^kuNdwCC^8RkuNdw
zCH*~#kuNdwB}Tr)$d^1jB}Tr)$d|V8Z{gp<&(N1VQzeGJ#L$-*`qCEuE&PTQl76G)
zc`F%ONQ`}nu`g}mXY5N`_?i3C7Jdf5w1uC^FKyvx^h*XQli`Tu`93lFB}Tu*=$9D%
z5~E*Y^h<^*5~E)-RFN3{5~E+z7n`>5Gy0`1{Q9kuVT{D^m$vXT{iQAZjDJagR??r9
zw(v6mrY-zi__y#IAWh7Gi5W04114s`q~9wUG)>Hai5W04119}o$q-30M3OvLCeM|L
z889&eCT76I44Ag@GXp05Wr-Ou8A?gafQcC}=|4;QFOp}`#0;49V<gX{sdy%3229L=
zi5W04114s`#0;320TVM|G8B`T0TVM|Vg^ixWs<?%#0;49g(POc#0;320TVM|(l?Uy
zjU<M^#1NPm0uw`EGWeUA0+W8cw2j|DaN5DYgP&0_8R|)lf@uf8e(uC9n3x6A4t|Eg
zw1b~%Fc}U?JNS3-GY}>Q!o)zBcJS}u-@(sJm<$;uX2Qfwn3xIE4*nhdJNS3-@8D-J
zObmvJ!7%OMXEIDohKb2AF&QQ%!^C8mm<*GC_rz$JcJS}u-@(6we+NJFVcNmZe3%SV
zB?iRAfS4E%lOe05-!kpsXBtR5`1RkXot?-RFH#wmQw3F06;)FW)lwbRQv)?p6E#x{
zwNe|kQwMcY7j;t)^->@8(*O<95Dn7^jnWv6(*#Y@6iw45x=dH-D$UR}nx*S>gKp9s
z&C>!c(h@Dx3a!!_t<wf=(iUwK{xbY!_{;E@;rF98xeUMl++2pg41XDZ{cD-OF0=6E
zGW=|OxePxmUoOMX&X>#Zv-IUM{A_)>3_ojMF2m2>m&@?8_~kPEY<{^6zdqw!hM(Op
zm*FqN&-Rzg@U#BqGW`1Wav6T(E4d7R8U8Z-`kJ%8=FAqD_4{Skz|0z$4NGSBz-$~Q
zm*Hm-%#04X48Oj?Y-}d$pU(QHb2)xPn7JH(IsS6|<@n3-m*X$TuMayj6K4I`xg38v
z{&M{K7;`y({fxOBe>r~rjkz2@BT6pEUyk26P-Z~P<@n3-m*Zzf%;osY@t5N-$6t=0
z$1pP!=5qYy_{;Hg31%+A%n_K&@t5N-$6t=0uP&G4*SDUT<uXTH=7`HoZ<*;WvruJ*
zs?6t>`P?#3TV}e-JZ+h$Emz>@WXl!!EAVr&WlpwSf!~mBHg=VH*D}*u=2puU_$%=1
zQ_U6lIny#TT4vwMEN8g_KPy@05z7_$4GCvsUzxcivvOsAt<0sB^~YwOs?0By4H;*4
zsLZ&N*`YG)Q)Ye2%qW@RDKj=@hNsNMl)0EP_g1dJ&$5&mm9l=_%&3$Zm2w6C3j7uL
z8J04`Qf64n3`?0|DKji(eZZM%DOcb(nJSx1l})C~#wN1KRJjtr$yC{7s%$7bn@p7}
z@taJQO{U5wQ)QE>vdL80WU8z`Ih#zCEAg99l})J1mG~?1SK_b4Z-Q0U=bTNl$|hN5
zldN(jeiN;-alu@P-$biiiN6wmC4Q5xawYys{7gC7<g0A*RW|u5n|zg-oN^`pO8h2a
z<x2dO_$%?7kd-U(SK_b4Ux~jGe<l7({KjB1lTT*y$yNBP@K@nCY@dx$<|_Oqb7e#L
zSwDHM!e52oaDHY8%7*lF75*yxRrss$SK((A%2oKQ@S6mdO#;g%f#oXvRrss$SK+V1
zUxlA7Ae$VPO%BT@hh-+C%w&{J4$F*2nPng|8fE?Tnb|0t9F`f5vhmYwa#%JwESnsb
zO%BT@hh>w)vdLk&3O`d)uEJl1pY0%5;jhA9g}(}a6@I3rT#cV`DOck+sVtjRmaFks
z<2Si1SL3h7UyZ*SKZ8?dVaV0^*%&fcdS-aaT<O^aw9NRF8J{xaQ?ABud?9nIXKwXu
z%r{r#H%Tp5<FCeVvRbajUyYw7B3I+D#$S!U8h<r@li4!!RA!#aCbZ>h{Hzn%#I|f=
zTV|oiCbnhc7@4^$SL0`}%GLO*@mJ%o#$S!U8h<tZYWy|$Yw*|Lufbn~zXrdtkZf{X
zHaRXcY2_OHHTaE>WN!Ox+&UX4$;Pg;v65Va-{iSmgTDrU4So~pvT?p_GF`60UxS~~
zE1OuC**&t!b(!HS*Wj<gZ#*#9;IF}7gTDs9@xfe!-xy)8!C!;lSYfWgUxU8}zX^Jo
zbtD@<%r*EK$TAyAuEAe}zXpE|{u=zo7jq5%8vHf*SxvI>#>_038P2kC$6SlQ7C-Y@
zW<$wl5#(C@wfK!m=34x<_-pYSx6fu5WU~u$E&f{kwfJlCGp=Rhm$?>yEq>#exfVY&
zTdu`#Of%Qwuf<=BzZSn43b_`)SqhnXG4lgt#<$E9ka+@fE&f{kEHSwje=Yu6{I&RN
z@f)+sd;^(pAlKqIewAzSGtOl`f?SKg7Jn^%o`PJ5-}HuDhrbTLX%4v#KZ9N7H^_DP
zO?k+5_*rjq9e&2UT!+67KR-gQ!(WG=EhpFEuft!5zYc#Letv~qho5I5*WqW~$#wYa
z@Ymt5!_Ui*>+sj%uft!5zYc#L{yO}1`0Mc3;WzCf8w<|Hf^!{y(=M_x;arE`^owlz
zMK)%a83VI%;%uBa*Ws_jZyYbz;jhDAho5&Mo1T&D@z>)wO(UD8k?Zl-<2PL+GZ*H1
z{KlGdJ^p(9_4w=Y*W)+-oa^z|<FCihi;?T`*W<6pUyr{YKZ9bf$8W4U*W<6pUyr{Y
ze?5NY#axfS9)CUldi?eH>+#p)ugA~hkxd)Prj2A?k6e%6xOc9{Z|plWKIVG-_4w=Y
z*W<6pUyr{Ye?5NV<GCJxJ^lv#4fq@IH{fr;-+;dXKLcfMz|UTl8}K*aZ@}MxzX5*(
z{s#OF_#5yy;Ag+e4fq@I8!ycb_#5yy;BUa+fWHAhLuPKkZ`w;X?IoM`l1+Q@qm+c-
z*nMum-+;dXzv(d9beL>9Og0@RH{fr;-+;dXe*^vo{0;b-JaYs72K){989p<^XKujH
z7n2+Cvxnsd{EhgTgL5PPM*PNhGh=9G49)Cgxe<RO{zm+b_#5#z;%~&?h`$kkBmPGG
zjrbe!GnD2={7j{}5q~3o=F;4VzY#xALT<$0h`$kkBmPGGd<wY{e<S`z{Ehe<@i*e<
z<;jis8}T>dZ^Yk-zY%{U{zm+b_?et@BYu8|+=!o{H8<jK#Lw888}YNm<wpEWuDJ<6
zqib%$--MsxH8Z^CCj3qKoA5KgX5-zt34as*Cj3qKoA9&NWyaXdTakGyGJ|YpwaZQT
zoA5W`XS&Wzv$+XB<7{ri--N#je-r*D{7v}zGIA6CCj3qKoA9&jWk%cFgue-Y6aFUr
zP57JeGvMYX{7v{7adQ)Xo~PV|zX^X6{wDl9P?_~FH{oa0%}w~3b#pWRX8cUMnJqB$
zN#$nz&G`AHax;FOsoad8Zz?zAZ^qw@zZris{$~8m_?z)J<8Q{_jK3LwGyZ1$&G?(~
zH{)-{-;BQ*KO16h#@~#;8Gkc=-mKh=zZrisem2G2jK3LwGk)Hc+>E~&e>47O{JdPb
z8GkeWX8a7uxfy>m{$~8m_?z)J<L7(H&G?(~x8QHV--4fEIkQ7%cF5d<zXd-7bLNrB
zJTkcjKc7r)!QX<v1%C_v7W^&vTkyBwXRyyL_*?L|;AghaE%;mT^VsAT{CsDbAv(9<
zZ^7S!pE)|SVP*#D+=8DMC%52l!QX<PX*##yZ^7S!pHC;Z;OEuJE%+I!a|`|!{4Mxf
z@VDUS)5tCOTkyBwZ^7S!zXg9Q{#N|0_*?O};%~*z6PH`@Gi7JqxZH~0gn`_OpGiBn
z;%~*@ioX?qEB;pet@wFDax4B;{H^#~@weh{#m{1znY(i<{#N|0_*?Pw-sM*Ot@vB<
zx8iTb--^E#e=Gi0{H^#~@weh{#ovm*6@M%KR{X8_Tk*HzZ^h3<o|(vVEB;pet@!zx
zax4B;{B8K#@VDV_!{3I#4SyScUclUjp9d<p;cvs=hQAGe8~!%@ZTQ>pv&`l;{B8K#
z@VDV_!{3I#4SyT{HvGJWxeb3C{x<w=_}lQe;cvs=hQAGe8~!%@Oz)W&F}LA2StGaM
zZ^Pe)zYTvIem=$AhQAGe8~!%@ZTQ)Da~u9P{B8K#@VDV_!_U^6+wiyJZ^z$`za4)&
zexAqNj=vp$JAVGh+>XB;KcjAL$KQ^>9e+FicKq%5+wr&KZ^z$`za2l{WNydLLY&+2
zx8rZe&&Z$K@wek|$IsNC+wr&KZ^z$`za4)&{&xKB_}lTf<8Q~`j=vp$JN|b3?fBdA
zx8rZe-;Tc>e>;8?b8<U=lXG%Aev=1sJN|b39r!!&ci`{9&*Gdr@OR+vz~6zt13&L)
z?!e!HzXN{<{to;d_&e}-;P1fSfxiQP2mTKH9r!!&ci`{9&p(<w@OR+vz~6zt1Ahnp
z4*VVXJMi<E<_`QF_&e}-;P1fSfxiPkKVI&@-+`YmFL&VQ&C4D5JMfzvkvs5r;P1fS
zfxiPk?_Tb}-+{jae<%J<{HE9DPW+wtJMr_k=1%;b_&f1;;_t-YiN6znC;m?So%lQP
zcjE8F--+Mkr_3LiJMr_#=1%;b_&f1;;_t-YiJx~icjE8F--*8yKQC?W#NUa(6F*;V
z?!@1T-$bR{iN6znC;m?SJh!<MKi_TUyUncWxf6dUejeQ1iN6znCw_k1+=;&vKVNR{
z#NUO#3x5~>F8p2iyYP46@50}OzYBjC{x1Ao_`C3T;qSuVg})1b7yd5%UHDBz$*l3Y
z3x5}W{@>h%zYBjCev_?o7yd5%UHDDB%3b)o@OR<o70zAwyYP46@50}OzY9OReeS~F
zg})1b7yd5%UHH53cj52C--VwqFL&YZ!rz6z3x5~>F8p2iyYTbv<!=1l_`C6&Jd?Ze
zcjIU2&)xXD@pt3r@5|lzyYYA9@5bMazZ*YKbMD69jlUazH-6IrayNcc0&+M0Zv5T&
zP0GpL_`C6U<L}1bjlUbesROwie>eVa{N4Dw@pt3z#@~&f$1r!}@5XPMLGH%ijo&1n
z+>O5*zsWwC4>5P+Hw__o<L}1bjlUazH~wz?J@|X@n-r9L@b}<1Wg++A@4?@LzXyL0
z{vP~2_<Qj8;P1iTgTDuV5B?tfJ@|X@_u%ir--Evge-Hj1{5|-4@b}>F!QX?w2R}b}
zHa#Nu;P1iTgTDuV5B?tfJ@|X@_u%ir--Evge-Hj1{5|-4@b}>F!QX?w2Y(O#9{fG{
zd+_(*@4?@LzXyLW{$BjO_<Qm9;_t=J&zpPk_u}ux-;2K&e=q)C{Jr>l@$<0fUi`ej
z*?f-Ni@z6tFaBQqz4&|a_u}ux&lj3`-g7U0zV~eYNAAVni@z6tFaBQqz4&|a_u}ux
z-;2K&e=q)C{Jf>P7k@AQUi`iId-3<;H*qod;_t=Zi@z6tFaBQqz4&|a_u}VM&VBg%
z@blm2KKy<7`|$VS@5A4R-^7&Mho6@<_u=Pf&3*X$@bfrllgM%({yzMD`1|np;qSxW
zhrbVhAO1f4{Lr}%e;@un{C)WQ@b}^G!{3L$4}TwizUgdoTIQY3efasOb07Xb{C)WQ
z@b}^G!{3L$55EaCxevd21i24C&u#9*--o{sKmTnuzaaPF@5A4ZzaM`;{(k)Z`1|qm
z=;nU>{Mxx6e?R_y{Qda*@%Q8J$KQ|NWS!iPzaM`;{(k)Z`1|qq<L}4ckG~(kNj<qA
ze?R_y{Qda*@%Q8J$KQ{?AAdjoe*FFT`NeZT{(k)Z`1|pjACdd<_v7!!-;ci^zX?OR
zA3tw#<}J>=#hJG__v7!!-;ci^zlp8c{E9q)e*pgg{sH_0_y_Qtbd(4158xlbKY)J#
z{{a30`~&z0@DJc0z(0V00RI5~0sI5_2k;NzAHY9=e*pgg{sH_0_y_P0;2*%x_ni5j
z^8o$<`~&#S_sD$Ec>w<a{sH_0_y_Qt2a*Tyn;4Y`@DJc0z;A9y9>C8#od@unFqH@J
z58xlbKY)J#{{a30{3cUnb4l_b{z3eM_y_S1;vd9sqE$B0Di7ix#6O6C5I^sJ9>hP0
ze-Qs5{z3eM_y_TuK$%U>%7gd^@ekrRQ7aGPAH;9&N;Y>T58^j@D-Yu5^Uj0#2k{T$
z=T*+;2joHggZKyW58@xhKZt)2zsZ?-5I=u-9>hP0e-Qs5{z3eM_y_S1;vd96h<_0O
zApSx8L->dA58)reKZJh>{}BEm{5<D*2>%fNA^b!5hwz(dmWS{U;pa)uL-<WR%R~5w
z@bjqWA^b!5hwu;KAHqL`e+WPSdLF_*gntM>FMA%sKZJh>{}BEm{6qM8-188A^Ca>R
z{vrHB`1!c=5dI<jL->dA58)reKZJh>{}6t2EAkM2li~ck6T&})e+d6DeiPL5F#ciu
z!}v{@%jRO_Vf@4Rhw%^NH@_$k;~&O9jDHyaF#ciu!}y2s591%kKaAg`yF84482>Q-
zVf@4Rhw%^NAI3k7e;EHT{$c#X_=oWi<2RQn591%kKa77E|1kbx{KNQ%@eku4#y^aI
z82>Q-Vf@4Rhw%^NAI5K9Ngl>OjDHyaF#ciuBlt(~kKiA{KZ1V*{|NpO{3G~B@Q>g(
z&mfQBAHi=ja2~-wf`0`62>ucLBlt(~kKiA{KZ1V*{|J6_6!HlE5&R?gNAR1okj;0=
z=4<5<{3G~B@Q>gh!9Rk31pf&B5&R?gNAQo}AHhF@e+2&se)Ar(3Cei{{|NpO{3G~B
z@Q>gh!9Rk31pf&B5&Wb0&6CKZ_($=N;x}g^kK!N2KZ<`8|0sU*DDo(N6QJ`b{!#p+
z_($=N;vdC7ihmUUD1H+yvw0VJ6#ppxQT(I$NAZv1H@_#3;vdC7ihmUUDE?9WqxeVh
zkK#9PBah-A#XpLF6#poGKdO*N@tcgDO~%fn_($=N;vdC7ihmUUDE?9W=6>W+{G<3s
z@sHv+zbKF4AHzR}e+>T^{xSSx_{Z>%;UB|4hTmMIJci%Ax;%z|4F4E@b4T+S{xSSx
z_{Z>%;UB|4hTkOdJcfS^{}_IAQ1Tdl^P2J){xSUKH{~(>WBAALkKrG~KZbt{{}}!;
z{A2jX@Q>ji!#{?94F4GZG5llr$MBEgAHzR}e+>T^{xSSx_{Z>%;UC9u&TAgWKaPJK
zzxl6u9RE1}as1=>$MKKjH;*un<2OGxkK-T5KaPJK|2Y0}{Nwn?@sHyl$3KpL9RE1}
zas1=>$MKKjAICqAe;off{&D={_{Z^&<2O$ykK-T5KaPJK|2Y0}{Nwn?@sHyl$8X|&
z9>+h9e;off{&D={_{Z^&;~&RAj(;5gIQ|Lz6Zp+5%M<t~@K4~Mz(0Y10{;Yl^Ct5I
z{t5gO_$TmB;Ge)hfqw%31pW#96Zj|ao1>N|@K4~Mz;C`$p1?nWe*(X`M|lFjIc(V+
zq&$It0{;a53H%fIC-6_;pTIwXe**sm{t5i%EM-3|kSFl_ae+L6e**sm{t5hkY#^H(
zmnZO>AD7Mf%;q@d3H%fIC-6_=pTzG+3GyWVN&J)e{XjvU#6O9D68|LrN&J)eC-G0>
zpTuv@U7o~0iGLEm`O?{s8stg*llUj`PvZ9@2YC{|A3DgB_$TpC;-ADniQkVO<VpOK
z_$TpC;`d_+*?g>QK2|nYHBaK7#6O9D68|LrN&J)eC-G0>pTs|je-i&B{z?3k_|0?8
zllaYb&6D^i@lWEP!f(!CHpe?p;h(}kg?|eF6#gmvQ~0OwPvM`!KZSn^{}g_6WwT#r
z$W!>I@cWg9JcZx<**t}R3jY-TDg0CTr|_Fso2T$k;h(}kh2Jkb<SG2-E@ty>^A!Fm
z{8RX+@K52N!as$73jY-TDg0CTr|?hV_sbA@3jY-TDg0CTr|?hVpTa+de+vH;e)A)<
zIlXxrzj?iR8viu@Y5aa(Et@-;r}0nYpT<9p-(1Q(jei=yIl$Q*;5?0g8viu@Y5ddp
zr}3LVm#6Vh|36jTOKxS2x@Kjv7Zhm&P0SQo@TC9<K>@@$=ifVgw;?Hk5DnrRujD51
z?1Y4*;2B;aYel@F2zz<XIi9dzM|Xeh|6~6j`~TSg$NoR||FQp%{eSG&*xw)f|JeV>
ze(l};vHy?#f9(HbzZUQQ*sndlKlcBz|BwBD?Eho`AN&8<|HuA6_W!Z}kNtn_|6@Oq
z`Tp3iS-wB^zu5m`zrOEY?0>QU#r_xjU+jOe|HXcN^S#*rV*iW%FZRFK|6>1({rb{-
zvH!*X7yDoAf3g3?{uldS?0>QU#r_xjU+jOe|Hb|n`(NzWr{0VGFZRFK|6>1({V(>v
z*#Bbxi~TS5zu5m`|BL;)#Cx&-#r_xjU+jOe|Hb|n`(NyTvH!*X7yDoA*D>CU{d(AY
zwV!KnulB#%|7!oM{jc`F+W%_*tNpL`zuNz5|EvA4_P^TyYX7VKT!eeI|JD9i`(N#U
zwg1)rSNmV>*M{G#{jc`F+W%_*tNpL`zuNz5|EvA4_LEKS)&5ueU+sUj|JD9i`=Lqq
zYX7VKulB#%|7!oM{jc`F+W%_*tNpL`zuNz5KXm(E?SHlZ)qb7&z1ja}|C{}9_P^Qx
zX8)W0Z}z|0|7QQ2{crZa+5cw$oBeP0zuEt0|C{}9_P^QxX8)W0Z}z|0|7QQ2{crZa
z+5cw$oBeP0zuEt0|C{}9_P^OrbGbMB-|T<0UvqkI_P^QxX8)W0Z}z|0|7QQ2{crYb
z?eES0H~Zi0f3yG1e*OKu+5cw$oBeP0zuEt8|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_
z+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~
z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g
z{qOd_+y8F=yZ!I>zuW&||A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F
z|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIP
z|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIPuV+8b!hP8PVgHByANFh9
z@5BC2`?c=(Y5%AFER*}R|I_|Y`)NY=X+QM)KJ5p8->3cX@B6g>)BaEUHPH8I|EK+*
z_J7*{Y5%AFK(qU_|I_|Y`#<gfwExq7{=;!o?$iEH`#<g1Qs1ZjpZ1fC?$iEH`#<gf
zwExroPy6-O_i6v9{krP|lHI5MpZ0&+uW`Ol`*r;HX+Nm^wAlwMKRVKV+W%$$m;GP%
zf7$<K|Cjw=_J7&`W&fA`U-onNkGp?g_J7&`W&fA`U-oOM@5_Gv%YE7ZW&fA`{DAwi
zUk`p?_J7&`W&fA`U-p05|7AZj=DzIzvj59|V$*%u|7HJ|{a^Nf+5ct#m;GP%>&x%U
z{xAE#?EkX=%l<F>zwBqv+?W0QnftQ;%l<F>zwH0A|I2>1!F}8RZU49Z-}Y0Y?%V!v
z`@ik~wx54+-}Zmo|84)b{oIcGw*TAyZ~MRP_kR-HxBcJtlcw(5{%`xg?f<s_+kUX!
zecR8(xo`Ww?f<s_+x~C+zwQ6F|J(j=`@ik~w*TAyZ~MRPXX)Ix{onR~+y8ApTj##*
z|F-|z{%`xg?f<r)xpT~&`?mkv{%`xg?dOi%kNrRPQ?c&H{vZ2UB==+gkNrRP|Jcv#
zxgYy~?EkU<$NnGtf9(IU|Hpoo&;8i{WB-r+l&$-*|HppD&;8i{WB-r+KlcCF|6~7;
z{Xh2q*#BeykNrRP|JeUy|BwAY_W#)bV?VX)e(e9T|Hu9x`zc=cWB-r+KlcCFPxZPV
z`+w~JvH!>ZANzmo|FK{HfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz
zPyc`V|I`1U{@;J0>&<@s|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-
zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T
z|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V
z|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd
z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-
z|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U
z{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va
z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz
z>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<
zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a
zpZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>Hkmv
zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w
z|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!
z|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^
z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv
z|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y
z{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<
z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b
z)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz
zPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-
zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T
z|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V
z|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd
z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-
z|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U
z{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va
z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz
z>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<
zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a
zpZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>Hkmv
zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w
z|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!
z|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^
z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv
z|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y
z{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<
z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b
z)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz
zPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-
zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T
z|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V
z|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd
z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-
z|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U
z{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va
z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs<r~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq
z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq
z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ
z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ
zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c
z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc
zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnx*Y@lG
z)BmUcPye6(zkhAN{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(zkg%D{y+VH`v3I*>HqsT_Ur%C|EK>?|DXOp{eSxZ^#AGq)BmUc
zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?
z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?
zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v)
z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(
zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp
z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n
zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH
z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D
z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ
z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq
z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq
z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ
z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ
zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c
z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc
zPye6(KmC9D|MdUq|9i0i!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT
z!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?AQON|4;v){y+VH`v3I*>HmAM
z|H1wT`ycGr|EK@&!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT!TtyP
z_5bPr)BmUcPye6(KmC9D{~qmswExlmNBi~v>HmAQU;p2u{g3uP+W%<(qy3NeKidCj
z|D*kn_CMPHX#b=AkM=*>|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*>
z|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*>|7ib{{ZIBk+5cq!ll@Qj
zKiU6e|C9at|MdUq|9i6k$^Iw%pX}HFr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ
z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGqd$#}C{%8B2?SHoa+5Tty
zpY4CP|Ji>1fBOHP?SHoa+5Tty_5bPr)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH
z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D
z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ
z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq
z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq
z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ
z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ
zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c
z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc
zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?
z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?
zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v)
z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(
zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp
z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n
zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH
z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D
z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ
z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq
z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq
z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ
z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ
zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c
z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc
zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?
z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?
zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v)
z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(
zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp
z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n
zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH
z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D
z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ
z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq
z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq
z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ
z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ
zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c
z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc
zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?
z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?
zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v)
z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(
zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp
z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n
zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH
z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D
z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ
z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIa
z#r}V>U;m%}KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ
z|EK>?|DXOp{eSxZ^#AGq`^)~n?AQON|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ
z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*
z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|L^VJe_6r+gaHTx5C$L&Kp2290AT>a0E7Vu
z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx
z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S
z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX
zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv
z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&
zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST
z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl
zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw
zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229
z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU
zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I
z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy
z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a
z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da
zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-
z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu1JJwu@Akjj|8749APhk7
z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^VY0SE&S1|SST7=SPU
zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I
z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy
z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da^kM&p{U7#!*pC4S0}uuv
z3_u_Df7t(F|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy9=Sf7<_P|EK+*_J7*{
zY5%AFpZ0&+|7riH{h#)K+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+|7riH{h#)K
z+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+j{yh+(5L;M_J7*{X+H)a3_uuwFaTiy
z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE)om;GP%f7$<K|Cjw=_J7&`W&fA`U-n}F!T|JT
z|Cjw=_J7%r0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0Q6=5m;GP%f7$<K
z|Cjw=_J7&`W&fA`U-p05|7HJ|{a^Nf+5ct#m;GP%f7$<K|Cjw=_J7&`W&fA`U-p05
z|7HJ|{a^Nf*^dDT0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST
z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl
zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw
zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229
z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU
zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I
z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy
z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaPpOf2ysc0Yn3c1`rJ(
z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2
zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo
zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaM~?`)L5t0HOgz1BeC?
z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1
zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(
z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2
zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0jb`1?r%hz1Z1AR0h4fM@{G
z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw
zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V
z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W
zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz
z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$
zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c
z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh
z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?
z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1
zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(
z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2
zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo
zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4
zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF
zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118
z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT
z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G
z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw
zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V
z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W
zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz
z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$
zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c
z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh
z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?
z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1
zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(
z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2
zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo
zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4
zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF
zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118
z0MP)V0Yn3c1`rJ(8bCCFXaLawMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR*uS=)1~3}HXaJ)Dj0P|o
zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt
zFdD#U0Q)!g(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|*iQo(4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|+D`)*4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e7Y
zll@QjKiU6eKMi0sfYAU(1K5-OPxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMG
zWdD=>Pxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMGWdD=>Pxe3A|78D@{ZIBk
z+5cq!ll@QjKiU6e|C9Ys_CMMGWdF1M&-Opt|7`!W{m=G4+y89;v;EKZKimIo|Fiwi
z_CMSIZ2z<U&-Opt|7`!W{m=G4+y89;v;EKZKimIo|Fiwi_CMSIY(EWPG=R|nMgtfP
zU^IZy07e5C4PZ2YJ=_0mKMi0sfYAU(1K6|u&-Opt|7`!W{m=G4+y89;v;EKZKimIo
z|Fiwi_CMSIY(EWPG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(1K59MKMi0sfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(E#>e+fM@+4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(
z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy
z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F
zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP
zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR
z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n
zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y
z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp
zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C
z4PZ2Y-~ZBuLWlqUABPVI*LfU%9Grb|cyjPM-r;}$$Kn5Q@TJ|s&;1Yo#eeJV@Bet`
zzx)5a<AJC7e}2}Fp6>s9@IU>>5B}%>_`$#P;NSkQKJ*_?`G3FT!9Skq|9$X}XZpAQ
z+kg0wfB(*Z_>lkbz=!;kt6RVS<fX5J=lUlvB!B<OSES#665s#+lQQ`CpY(0N|0D$Y
z{b%0-{Qk4gd%yqeAN}t?3lV?+Sxxi%&+fnd{bx4_I{f)he*eYg8i)VU!3nV6fAN{;
z_g}>94i64`o!@^E2>kx#(*5sW&V&B`<%HAkU(V7v_`Uq)e22q-<KR2W-@knN{ryXU
z{ri{B>-R6=(C=UW&Y$vM?$!PM%S}$df4OVr_b<18{Qk|k>fgUP3HkdsXJvl>=4Ha~
z-~7A#{TmM8_it{(|NWa=!hiqf-VO)9_}^Vw|NXnmln(!kgNuR=|9H>o_20icC-D1s
zU-W<f?kmOb-+c=H{kvrK_wQP;-@kwRHU92a_20j{-}?9OZch3A<BH1fAD0Jy|2R+o
z`^QPv-#^ak{r>T-*6$zROZ@)v;_>$nQ^LV-?a!y@`}2?A+8=jl|Ne11?e8D=6aD^i
z1Ih1;E4aTePV@Y}_`3W1A~*ki5##>8xCQ<9#l2m>FK#&cef3Ju!5`SG&q2Sh`ikFI
zciR5Gxor6R=9<p$n~NR4Z_b$fzWFxz_swh3-?vYH&)!`%{(X0O_4nQNliznI&VS#X
VIs1M0GVDM7SO3|+`p^H9{|m$tW0n8_

literal 0
HcmV?d00001

diff --git a/transformers/tests/tokenization_albert_test.py b/transformers/tests/tokenization_albert_test.py
new file mode 100644
index 0000000000..dd63f6756b
--- /dev/null
+++ b/transformers/tests/tokenization_albert_test.py
@@ -0,0 +1,78 @@
+# coding=utf-8
+# Copyright 2019 Hugging Face inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+import unittest
+
+from transformers.tokenization_albert import (AlbertTokenizer, SPIECE_UNDERLINE)
+
+from .tokenization_tests_commons import CommonTestCases
+
+SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                    'fixtures/30k-clean.model')
+
+class AlbertTokenizationTest(CommonTestCases.CommonTokenizerTester):
+
+    tokenizer_class = AlbertTokenizer
+
+    def setUp(self):
+        super(AlbertTokenizationTest, self).setUp()
+
+        # We have a SentencePiece fixture for testing
+        tokenizer = AlbertTokenizer(SAMPLE_VOCAB)
+        tokenizer.save_pretrained(self.tmpdirname)
+
+    def get_tokenizer(self, **kwargs):
+        return AlbertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
+
+    def get_input_output_texts(self):
+        input_text = u"this is a test"
+        output_text = u"this is a test"
+        return input_text, output_text
+
+
+    def test_full_tokenizer(self):
+        tokenizer = AlbertTokenizer(SAMPLE_VOCAB, keep_accents=True)
+
+        tokens = tokenizer.tokenize(u'This is a test')
+        self.assertListEqual(tokens, [u'▁this', u'▁is', u'▁a', u'▁test'])
+
+        self.assertListEqual(
+            tokenizer.convert_tokens_to_ids(tokens), [48, 25, 21, 1289])
+
+        tokens = tokenizer.tokenize(u"I was born in 92000, and this is falsé.")
+        self.assertListEqual(tokens, [u'▁i', u'▁was', u'▁born', u'▁in', u'▁9', u'2000', u',', u'▁and', u'▁this', u'▁is', u'▁fal', u's', u'é', u'.'])
+        ids = tokenizer.convert_tokens_to_ids(tokens)
+        self.assertListEqual(ids, [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9])
+
+        back_tokens = tokenizer.convert_ids_to_tokens(ids)
+        self.assertListEqual(back_tokens, ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.'])
+
+    def test_sequence_builders(self):
+        tokenizer = AlbertTokenizer(SAMPLE_VOCAB)
+
+        text = tokenizer.encode("sequence builders")
+        text_2 = tokenizer.encode("multi-sequence build")
+
+        encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
+        encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
+
+        assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
+        assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_2 + [tokenizer.sep_token_id]
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py
index f2e37222f6..0785e55ad2 100644
--- a/transformers/tokenization_albert.py
+++ b/transformers/tokenization_albert.py
@@ -8,6 +8,7 @@ from shutil import copyfile
 
 logger = logging.getLogger(__name__)
 
+VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'}
 SPIECE_UNDERLINE = u'▁'
 
 class AlbertTokenizer(PreTrainedTokenizer):
@@ -16,12 +17,12 @@ class AlbertTokenizer(PreTrainedTokenizer):
 
             - requires `SentencePiece <https://github.com/google/sentencepiece>`_
     """
-    # vocab_files_names = VOCAB_FILES_NAMES
+    vocab_files_names = VOCAB_FILES_NAMES
     # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
     # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, vocab_file,
-                 do_lower_case=False, remove_space=True, keep_accents=False,
+                 do_lower_case=True, remove_space=True, keep_accents=False,
                  bos_token="[CLS]", eos_token="[SEP]", unk_token="<unk>", sep_token="[SEP]",
                  pad_token="<pad>", cls_token="[CLS]", mask_token="[MASK]>", **kwargs):
         super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token,
@@ -142,15 +143,15 @@ class AlbertTokenizer(PreTrainedTokenizer):
         """
         Build model inputs from a sequence or a pair of sequence for sequence classification tasks
         by concatenating and adding special tokens.
-        A RoBERTa sequence has the following format:
-            single sequence: <s> X </s>
-            pair of sequences: <s> A </s></s> B </s>
+        An ALBERT sequence has the following format:
+            single sequence: [CLS] X [SEP]
+            pair of sequences: [CLS] A [SEP] B [SEP]
         """
         sep = [self.sep_token_id]
         cls = [self.cls_token_id]
         if token_ids_1 is None:
-            return token_ids_0 + sep + cls
-        return token_ids_0 + sep + token_ids_1 + sep + cls
+            return cls + token_ids_0 + sep
+        return cls + token_ids_0 + sep + token_ids_1 + sep
 
     def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
         """
@@ -175,25 +176,24 @@ class AlbertTokenizer(PreTrainedTokenizer):
             return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
 
         if token_ids_1 is not None:
-            return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1]
-        return ([0] * len(token_ids_0)) + [1, 1]
+            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1]
 
     def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
         """
         Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
-        A BERT sequence pair mask has the following format:
-        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2
-        | first sequence    | second sequence     | CLS segment ID
+        An ALBERT sequence pair mask has the following format:
+        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 
+        | first sequence    | second sequence     
         
         if token_ids_1 is None, only returns the first portion of the mask (0's).
         """
         sep = [self.sep_token_id]
         cls = [self.cls_token_id]
-        cls_segment_id = [2]
 
         if token_ids_1 is None:
-            return len(token_ids_0 + sep + cls) * [0]
-        return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id
+            return len(cls + token_ids_0 + sep) * [0]
+        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]
 
     def save_vocabulary(self, save_directory):
         """ Save the sentencepiece vocabulary (copy original file) and special tokens file
diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py
index a4f1a6e3ba..c01fbbbeeb 100644
--- a/transformers/tokenization_xlnet.py
+++ b/transformers/tokenization_xlnet.py
@@ -185,9 +185,9 @@ class XLNetTokenizer(PreTrainedTokenizer):
         """
         Build model inputs from a sequence or a pair of sequence for sequence classification tasks
         by concatenating and adding special tokens.
-        A RoBERTa sequence has the following format:
-            single sequence: <s> X </s>
-            pair of sequences: <s> A </s></s> B </s>
+        An XLNet sequence has the following format:
+            single sequence: X <sep> <cls>
+            pair of sequences: A <sep> B <sep> <cls>
         """
         sep = [self.sep_token_id]
         cls = [self.cls_token_id]
@@ -224,7 +224,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
     def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
         """
         Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
-        A BERT sequence pair mask has the following format:
+        An XLNet sequence pair mask has the following format:
         0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2
         | first sequence    | second sequence     | CLS segment ID
         

From 1e5b31c3881cb1313216ce0f3cffae89b0845d4f Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 20:25:32 +0000
Subject: [PATCH 211/269] Several fixes and improvements

---
 transformers/modeling_albert.py               |  36 +++++++++---------
 .../{30k-clean.model => spiece.model}         | Bin
 transformers/tokenization_albert.py           |   2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)
 rename transformers/tests/fixtures/{30k-clean.model => spiece.model} (100%)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index ad8b979cef..371a2e535c 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -7,6 +7,7 @@ import torch.nn as nn
 from torch.nn import CrossEntropyLoss
 from transformers.configuration_albert import AlbertConfig
 from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new
+from transformers.modeling_utils import PreTrainedModel
 from .file_utils import add_start_docstrings
 
 logger = logging.getLogger(__name__)
@@ -37,18 +38,17 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
         print(name)
     
     for name, array in zip(names, arrays):
-        print(name)
-        og = name
+        original_name = name
         name = name.replace("ffn_1", "ffn")
         name = name.replace("ffn/intermediate/output", "ffn_output")
         name = name.replace("attention_1", "attention")   
-        name = name.replace("cls/predictions/transform", "predictions")
-        name = name.replace("LayerNorm_1", "attention/LayerNorm")    
+        name = name.replace("cls/predictions", "predictions")
+        name = name.replace("transform/", "")
+        name = name.replace("LayerNorm_1", "full_layer_layer_norm")    
+        name = name.replace("LayerNorm", "attention/LayerNorm")    
         name = name.replace("inner_group_", "albert_layers/") 
         name = name.replace("group_", "albert_layer_groups/")    
         name = name.split('/')
-
-        print(name)
         pointer = model
         for m_name in name:
             if re.fullmatch(r'[A-Za-z]+_\d+', m_name):
@@ -78,13 +78,12 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
             pointer = getattr(pointer, 'weight')
         elif m_name == 'kernel':
             array = np.transpose(array)
-            print("transposed")
         try:
             assert pointer.shape == array.shape
         except AssertionError as e:
             e.args += (pointer.shape, array.shape)
             raise
-        print("Initialize PyTorch weight {} from {}".format(name, og))
+        print("Initialize PyTorch weight {} from {}".format(name, original_name))
         pointer.data = torch.from_numpy(array)
 
     return model
@@ -177,9 +176,9 @@ class AlbertAttention(BertSelfAttention):
         b = self.dense.bias
 
         projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
-        projected_context_layer = self.dropout(projected_context_layer)
-        layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer)
-        return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask
+        projected_context_layer_dropout = self.dropout(projected_context_layer)
+        layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout)
+        return layernormed_context_layer
 
 
 class AlbertLayer(nn.Module):
@@ -187,17 +186,17 @@ class AlbertLayer(nn.Module):
         super(AlbertLayer, self).__init__()
         
         self.config = config
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.full_layer_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.attention = AlbertAttention(config)
         self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        attention_output = self.attention(hidden_states, attention_mask)[0]
+        attention_output = self.attention(hidden_states, attention_mask)
         ffn_output = self.ffn(attention_output)
         ffn_output = gelu_new(ffn_output)
         ffn_output = self.ffn_output(ffn_output)
-        hidden_states = self.LayerNorm(ffn_output + attention_output)
+        hidden_states = self.full_layer_layer_norm(ffn_output + attention_output)
 
         return hidden_states
 
@@ -352,16 +351,17 @@ class AlbertModel(BertModel):
         encoder_outputs = self.encoder(embedding_output,
                                        extended_attention_mask,
                                        head_mask=head_mask)
+
         sequence_output = encoder_outputs[0]
-    
+
         pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0]))
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
+        outputs = (sequence_output, pooled_output) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
         return outputs
 
 
 @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
-class AlbertForMaskedLM(nn.Module):
+class AlbertForMaskedLM(PreTrainedModel):
     r"""
         **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
             Labels for computing the masked language modeling loss.
@@ -384,7 +384,7 @@ class AlbertForMaskedLM(nn.Module):
     """
     
     def __init__(self, config):
-        super(AlbertForMaskedLM, self).__init__()
+        super(AlbertForMaskedLM, self).__init__(config)
 
         self.config = config
         self.bert = AlbertModel(config)
diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/spiece.model
similarity index 100%
rename from transformers/tests/fixtures/30k-clean.model
rename to transformers/tests/fixtures/spiece.model
diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py
index 0785e55ad2..7b16bb573f 100644
--- a/transformers/tokenization_albert.py
+++ b/transformers/tokenization_albert.py
@@ -8,7 +8,7 @@ from shutil import copyfile
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'}
+VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'}
 SPIECE_UNDERLINE = u'▁'
 
 class AlbertTokenizer(PreTrainedTokenizer):

From 5680a1106302b1ebeb960de0700d6379c0aeef5c Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 20:42:49 +0000
Subject: [PATCH 212/269] Activation function managed from the config file

---
 transformers/configuration_albert.py | 2 +-
 transformers/modeling_albert.py      | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py
index c86c9565cb..15437dbbea 100644
--- a/transformers/configuration_albert.py
+++ b/transformers/configuration_albert.py
@@ -16,7 +16,7 @@ class AlbertConfig(PretrainedConfig):
                  intermediate_size=16384,
                  inner_group_num=1,
                  down_scale_factor=1,
-                 hidden_act="gelu",
+                 hidden_act="gelu_new",
                  hidden_dropout_prob=0,
                  attention_probs_dropout_prob=0,
                  max_position_embeddings=512,
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 371a2e535c..7e9f7f1c46 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -6,7 +6,7 @@ import torch
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss
 from transformers.configuration_albert import AlbertConfig
-from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new
+from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN
 from transformers.modeling_utils import PreTrainedModel
 from .file_utils import add_start_docstrings
 
@@ -190,11 +190,12 @@ class AlbertLayer(nn.Module):
         self.attention = AlbertAttention(config)
         self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) 
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.activation = ACT2FN[config.hidden_act]
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
         attention_output = self.attention(hidden_states, attention_mask)
         ffn_output = self.ffn(attention_output)
-        ffn_output = gelu_new(ffn_output)
+        ffn_output = self.activation(ffn_output)
         ffn_output = self.ffn_output(ffn_output)
         hidden_states = self.full_layer_layer_norm(ffn_output + attention_output)
 
@@ -392,6 +393,7 @@ class AlbertForMaskedLM(PreTrainedModel):
         self.bias = nn.Parameter(torch.zeros(config.vocab_size))
         self.dense = nn.Linear(config.hidden_size, config.embedding_size)
         self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size)
+        self.activation = ACT2FN[config.hidden_act]
 
     def tie_weights(self):
         """ Make sure we are sharing the input and output embeddings.
@@ -405,7 +407,7 @@ class AlbertForMaskedLM(PreTrainedModel):
         outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)
         sequence_outputs = outputs[0]
         hidden_states = self.dense(sequence_outputs)
-        hidden_states = gelu_new(hidden_states)
+        hidden_states = self.activation(hidden_states)
         hidden_states = self.LayerNorm(hidden_states)
         prediction_scores = self.word_embeddings(hidden_states)
 

From ce9eade29c75fa676ac528d1fe21d9f4ac3c5622 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 20:50:44 +0000
Subject: [PATCH 213/269] Initializer range using BertPreTrainedModel

---
 transformers/modeling_albert.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 7e9f7f1c46..b45208b696 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -6,8 +6,7 @@ import torch
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss
 from transformers.configuration_albert import AlbertConfig
-from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN
-from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN
 from .file_utils import add_start_docstrings
 
 logger = logging.getLogger(__name__)
@@ -362,7 +361,7 @@ class AlbertModel(BertModel):
 
 
 @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
-class AlbertForMaskedLM(PreTrainedModel):
+class AlbertForMaskedLM(BertPreTrainedModel):
     r"""
         **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
             Labels for computing the masked language modeling loss.

From 25a31953e8820eb5c88d8ad35ee547efccfe577c Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 21:18:06 +0000
Subject: [PATCH 214/269] Output Attentions + output hidden states

---
 transformers/modeling_albert.py | 58 ++++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 9 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index b45208b696..52cab2ea69 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -105,6 +105,7 @@ class AlbertAttention(BertSelfAttention):
     def __init__(self, config):
         super(AlbertAttention, self).__init__(config)
 
+        self.output_attentions = config.output_attentions
         self.num_attention_heads = config.num_attention_heads
         self.hidden_size = config.hidden_size 
         self.attention_head_size = config.hidden_size // config.num_attention_heads
@@ -177,7 +178,7 @@ class AlbertAttention(BertSelfAttention):
         projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
         projected_context_layer_dropout = self.dropout(projected_context_layer)
         layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout)
-        return layernormed_context_layer
+        return (layernormed_context_layer, attention_probs) if self.output_attentions else (layernormed_context_layer,)
 
 
 class AlbertLayer(nn.Module):
@@ -193,25 +194,45 @@ class AlbertLayer(nn.Module):
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
         attention_output = self.attention(hidden_states, attention_mask)
-        ffn_output = self.ffn(attention_output)
+        ffn_output = self.ffn(attention_output[0])
         ffn_output = self.activation(ffn_output)
         ffn_output = self.ffn_output(ffn_output)
-        hidden_states = self.full_layer_layer_norm(ffn_output + attention_output)
+        hidden_states = self.full_layer_layer_norm(ffn_output + attention_output[0])
 
-        return hidden_states
+        return (hidden_states,) + attention_output[1:]  # add attentions if we output them
 
 
 class AlbertLayerGroup(nn.Module):
     def __init__(self, config):
         super(AlbertLayerGroup, self).__init__()
         
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
         self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)])
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        for albert_layer in self.albert_layers:
-            hidden_states = albert_layer(hidden_states, attention_mask, head_mask)
+        layer_hidden_states = ()
+        layer_attentions = ()
 
-        return hidden_states
+        for albert_layer in self.albert_layers:
+            if self.output_hidden_states:
+                layer_hidden_states = layer_hidden_states + (hidden_states,)
+
+            layer_output = albert_layer(hidden_states, attention_mask, head_mask)
+            hidden_states = layer_output[0]
+
+            if self.output_attentions:
+                layer_attentions = layer_attentions + (layer_output[1],)
+
+        if self.output_hidden_states:
+                layer_hidden_states = layer_hidden_states + (hidden_states,)
+
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (layer_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (layer_attentions,)
+        return outputs  # last-layer hidden state, (layer hidden states), (layer attentions)
 
 
 class AlbertTransformer(nn.Module):
@@ -227,11 +248,30 @@ class AlbertTransformer(nn.Module):
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
         hidden_states = self.embedding_hidden_mapping_in(hidden_states)
 
+        all_attentions = ()
+
+        if self.output_hidden_states:
+            all_hidden_states = (hidden_states,)
+
         for layer_idx in range(self.config.num_hidden_layers):
             group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups)
-            hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask)  
+            layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask)  
         
-        return (hidden_states,)
+            hidden_states = layer_group_output[0]
+
+            if self.output_attentions:
+                all_attentions = all_attentions + layer_group_output[1]
+
+            if self.output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+        
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (all_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (all_attentions,)
+        return outputs  # last-layer hidden state, (all hidden states), (all attentions)
 
 
 ALBERT_START_DOCSTRING = r"""    The ALBERT model was proposed in

From 870320a24e28f187d0dfd10b82c4d60d5269374d Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 30 Oct 2019 22:30:21 +0000
Subject: [PATCH 215/269] Early tests

---
 transformers/modeling_albert.py            |  59 +++----
 transformers/tests/modeling_albert_test.py | 191 +++++++++++++++++++++
 2 files changed, 219 insertions(+), 31 deletions(-)
 create mode 100644 transformers/tests/modeling_albert_test.py

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 52cab2ea69..f906352311 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -11,6 +11,15 @@ from .file_utils import add_start_docstrings
 
 logger = logging.getLogger(__name__)
 
+
+ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
+    'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin",
+    'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin",
+    'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin",
+    'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin",
+}
+
+
 def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
     """ Load tf checkpoints in a pytorch model."""
     try:
@@ -39,6 +48,7 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
     for name, array in zip(names, arrays):
         original_name = name
         name = name.replace("ffn_1", "ffn")
+        name = name.replace("/bert/", "/albert/")
         name = name.replace("ffn/intermediate/output", "ffn_output")
         name = name.replace("attention_1", "attention")   
         name = name.replace("cls/predictions", "predictions")
@@ -114,29 +124,6 @@ class AlbertAttention(BertSelfAttention):
         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.pruned_heads = set()
 
-    def prune_heads(self, heads):
-        if len(heads) == 0:
-            return
-        mask = torch.ones(self.num_attention_heads, self.attention_head_size)
-        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
-        for head in heads:
-            # Compute how many pruned heads are before the head and move the index accordingly
-            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
-            mask[head] = 0
-        mask = mask.view(-1).contiguous().eq(1)
-        index = torch.arange(len(mask))[mask].long()
-
-        # Prune linear layers
-        self.query = prune_linear_layer(self.query, index)
-        self.key = prune_linear_layer(self.key, index)
-        self.value = prune_linear_layer(self.value, index)
-        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
-
-        # Update hyper params and store pruned heads
-        self.num_attention_heads = self.num_attention_heads - len(heads)
-        self.all_head_size = self.attention_head_size * self.num_attention_heads
-        self.pruned_heads = self.pruned_heads.union(heads)
-
     def forward(self, input_ids, attention_mask=None, head_mask=None):
         mixed_query_layer = self.query(input_ids)
         mixed_key_layer = self.key(input_ids)
@@ -225,7 +212,7 @@ class AlbertLayerGroup(nn.Module):
                 layer_attentions = layer_attentions + (layer_output[1],)
 
         if self.output_hidden_states:
-                layer_hidden_states = layer_hidden_states + (hidden_states,)
+            layer_hidden_states = layer_hidden_states + (hidden_states,)
 
         outputs = (hidden_states,)
         if self.output_hidden_states:
@@ -367,6 +354,8 @@ class AlbertModel(BertModel):
         self.pooler = nn.Linear(config.hidden_size, config.hidden_size)
         self.pooler_activation = nn.Tanh()
 
+        self.init_weights()
+
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
@@ -422,33 +411,41 @@ class AlbertForMaskedLM(BertPreTrainedModel):
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
     """
-    
+
+    config_class = AlbertConfig
+    pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    load_tf_weights = load_tf_weights_in_albert
+    base_model_prefix = "albert"
+
     def __init__(self, config):
         super(AlbertForMaskedLM, self).__init__(config)
 
         self.config = config
-        self.bert = AlbertModel(config)
+        self.albert = AlbertModel(config)
         self.LayerNorm = nn.LayerNorm(config.embedding_size)
         self.bias = nn.Parameter(torch.zeros(config.vocab_size))
         self.dense = nn.Linear(config.hidden_size, config.embedding_size)
-        self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size)
+        self.decoder = nn.Linear(config.embedding_size, config.vocab_size)
         self.activation = ACT2FN[config.hidden_act]
 
+        self.init_weights()
+        self.tie_weights()
+
     def tie_weights(self):
         """ Make sure we are sharing the input and output embeddings.
             Export to TorchScript can't handle parameter sharing so we are cloning them instead.
         """
-        self._tie_or_clone_weights(self.classifier.word_embeddings,
-                                   self.transformer.embeddings.word_embeddings)
+        self._tie_or_clone_weights(self.decoder,
+                                   self.albert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None):
-        outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)
+        outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)
         sequence_outputs = outputs[0]
         hidden_states = self.dense(sequence_outputs)
         hidden_states = self.activation(hidden_states)
         hidden_states = self.LayerNorm(hidden_states)
-        prediction_scores = self.word_embeddings(hidden_states)
+        prediction_scores = self.decoder(hidden_states)
 
         outputs = (prediction_scores,) + outputs[2:]  # Add hidden states and attention if they are here
         if masked_lm_labels is not None:
diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py
new file mode 100644
index 0000000000..46a2eeb729
--- /dev/null
+++ b/transformers/tests/modeling_albert_test.py
@@ -0,0 +1,191 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+import shutil
+import pytest
+
+from transformers import is_torch_available
+
+from .modeling_common_test import (CommonTestCases, ids_tensor)
+from .configuration_common_test import ConfigTester
+
+if is_torch_available():
+    from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM)
+    from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+else:
+    pytestmark = pytest.mark.skip("Require Torch")
+
+
+class AlbertModelTest(CommonTestCases.CommonModelTester):
+
+    all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else ()
+    test_pruning = False
+    test_head_masking = False
+
+    class AlbertModelTester(object):
+
+        def __init__(self,
+                     parent,
+                     batch_size=13,
+                     seq_length=7,
+                     is_training=True,
+                     use_input_mask=True,
+                     use_token_type_ids=True,
+                     use_labels=True,
+                     vocab_size=99,
+                     hidden_size=32,
+                     num_hidden_layers=5,
+                     num_attention_heads=4,
+                     intermediate_size=37,
+                     hidden_act="gelu",
+                     hidden_dropout_prob=0.1,
+                     attention_probs_dropout_prob=0.1,
+                     max_position_embeddings=512,
+                     type_vocab_size=16,
+                     type_sequence_label_size=2,
+                     initializer_range=0.02,
+                     num_labels=3,
+                     num_choices=4,
+                     scope=None,
+                    ):
+            self.parent = parent
+            self.batch_size = batch_size
+            self.seq_length = seq_length
+            self.is_training = is_training
+            self.use_input_mask = use_input_mask
+            self.use_token_type_ids = use_token_type_ids
+            self.use_labels = use_labels
+            self.vocab_size = vocab_size
+            self.hidden_size = hidden_size
+            self.num_hidden_layers = num_hidden_layers
+            self.num_attention_heads = num_attention_heads
+            self.intermediate_size = intermediate_size
+            self.hidden_act = hidden_act
+            self.hidden_dropout_prob = hidden_dropout_prob
+            self.attention_probs_dropout_prob = attention_probs_dropout_prob
+            self.max_position_embeddings = max_position_embeddings
+            self.type_vocab_size = type_vocab_size
+            self.type_sequence_label_size = type_sequence_label_size
+            self.initializer_range = initializer_range
+            self.num_labels = num_labels
+            self.num_choices = num_choices
+            self.scope = scope
+
+        def prepare_config_and_inputs(self):
+            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
+
+            input_mask = None
+            if self.use_input_mask:
+                input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
+
+            token_type_ids = None
+            if self.use_token_type_ids:
+                token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
+
+            sequence_labels = None
+            token_labels = None
+            choice_labels = None
+            if self.use_labels:
+                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
+                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
+                choice_labels = ids_tensor([self.batch_size], self.num_choices)
+
+            config = AlbertConfig(
+                vocab_size_or_config_json_file=self.vocab_size,
+                hidden_size=self.hidden_size,
+                num_hidden_layers=self.num_hidden_layers,
+                num_attention_heads=self.num_attention_heads,
+                intermediate_size=self.intermediate_size,
+                hidden_act=self.hidden_act,
+                hidden_dropout_prob=self.hidden_dropout_prob,
+                attention_probs_dropout_prob=self.attention_probs_dropout_prob,
+                max_position_embeddings=self.max_position_embeddings,
+                type_vocab_size=self.type_vocab_size,
+                initializer_range=self.initializer_range)
+
+            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+
+        def check_loss_output(self, result):
+            self.parent.assertListEqual(
+                list(result["loss"].size()),
+                [])
+
+        def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = AlbertModel(config=config)
+            model.eval()
+            sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
+            sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
+            sequence_output, pooled_output = model(input_ids)
+
+            result = {
+                "sequence_output": sequence_output,
+                "pooled_output": pooled_output,
+            }
+            self.parent.assertListEqual(
+                list(result["sequence_output"].size()),
+                [self.batch_size, self.seq_length, self.hidden_size])
+            self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size])
+
+
+        def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = AlbertForMaskedLM(config=config)
+            model.eval()
+            loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels)
+            result = {
+                "loss": loss,
+                "prediction_scores": prediction_scores,
+            }
+            self.parent.assertListEqual(
+                list(result["prediction_scores"].size()),
+                [self.batch_size, self.seq_length, self.vocab_size])
+            self.check_loss_output(result)
+
+
+        def prepare_config_and_inputs_for_common(self):
+            config_and_inputs = self.prepare_config_and_inputs()
+            (config, input_ids, token_type_ids, input_mask,
+             sequence_labels, token_labels, choice_labels) = config_and_inputs
+            inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
+            return config, inputs_dict
+
+    def setUp(self):
+        self.model_tester = AlbertModelTest.AlbertModelTester(self)
+        self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_albert_model(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_model(*config_and_inputs)
+
+    def test_for_masked_lm(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs)
+
+    @pytest.mark.slow
+    def test_model_from_pretrained(self):
+        cache_dir = "/tmp/transformers_test/"
+        for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
+            model = AlbertModel.from_pretrained(model_name, cache_dir=cache_dir)
+            shutil.rmtree(cache_dir)
+            self.assertIsNotNone(model)
+
+if __name__ == "__main__":
+    unittest.main()

From c14a22272f3fc17bb2eaeca62986c31a7d26bc85 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 31 Oct 2019 14:04:10 +0000
Subject: [PATCH 216/269] ALBERT passes all tests

---
 transformers/configuration_albert.py           | 4 +---
 transformers/modeling_albert.py                | 9 +++------
 transformers/tests/tokenization_albert_test.py | 2 +-
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py
index 15437dbbea..04f9fa8d60 100644
--- a/transformers/configuration_albert.py
+++ b/transformers/configuration_albert.py
@@ -7,7 +7,7 @@ class AlbertConfig(PretrainedConfig):
     """
 
     def __init__(self,
-                 vocab_size_or_config_json_file,
+                 vocab_size_or_config_json_file=30000,
                  embedding_size=128,
                  hidden_size=4096,
                  num_hidden_layers=12,
@@ -15,7 +15,6 @@ class AlbertConfig(PretrainedConfig):
                  num_attention_heads=64,
                  intermediate_size=16384,
                  inner_group_num=1,
-                 down_scale_factor=1,
                  hidden_act="gelu_new",
                  hidden_dropout_prob=0,
                  attention_probs_dropout_prob=0,
@@ -61,7 +60,6 @@ class AlbertConfig(PretrainedConfig):
         self.num_hidden_groups = num_hidden_groups
         self.num_attention_heads = num_attention_heads
         self.inner_group_num = inner_group_num
-        self.down_scale_factor = down_scale_factor
         self.hidden_act = hidden_act
         self.intermediate_size = intermediate_size
         self.hidden_dropout_prob = hidden_dropout_prob
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index f906352311..9bb38dead9 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -202,17 +202,14 @@ class AlbertLayerGroup(nn.Module):
         layer_attentions = ()
 
         for albert_layer in self.albert_layers:
-            if self.output_hidden_states:
-                layer_hidden_states = layer_hidden_states + (hidden_states,)
-
             layer_output = albert_layer(hidden_states, attention_mask, head_mask)
             hidden_states = layer_output[0]
 
             if self.output_attentions:
                 layer_attentions = layer_attentions + (layer_output[1],)
 
-        if self.output_hidden_states:
-            layer_hidden_states = layer_hidden_states + (hidden_states,)
+            if self.output_hidden_states:
+                layer_hidden_states = layer_hidden_states + (hidden_states,)
 
         outputs = (hidden_states,)
         if self.output_hidden_states:
@@ -247,7 +244,7 @@ class AlbertTransformer(nn.Module):
             hidden_states = layer_group_output[0]
 
             if self.output_attentions:
-                all_attentions = all_attentions + layer_group_output[1]
+                all_attentions = all_attentions + layer_group_output[-1]
 
             if self.output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
diff --git a/transformers/tests/tokenization_albert_test.py b/transformers/tests/tokenization_albert_test.py
index dd63f6756b..59eb3bceb0 100644
--- a/transformers/tests/tokenization_albert_test.py
+++ b/transformers/tests/tokenization_albert_test.py
@@ -22,7 +22,7 @@ from transformers.tokenization_albert import (AlbertTokenizer, SPIECE_UNDERLINE)
 from .tokenization_tests_commons import CommonTestCases
 
 SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                    'fixtures/30k-clean.model')
+                    'fixtures/spiece.model')
 
 class AlbertTokenizationTest(CommonTestCases.CommonTokenizerTester):
 

From b21402fc86257feca05ab050d061e15441a49929 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 31 Oct 2019 14:19:31 +0000
Subject: [PATCH 217/269] Python 2 tests + licence

---
 transformers/modeling_albert.py     | 16 ++++++++++++++++
 transformers/tokenization_albert.py | 19 ++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 9bb38dead9..b6d1291725 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -1,4 +1,20 @@
 
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch ALBERT model. """
+
 import os
 import math
 import logging
diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py
index 7b16bb573f..7cba99b9e4 100644
--- a/transformers/tokenization_albert.py
+++ b/transformers/tokenization_albert.py
@@ -1,4 +1,21 @@
-
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization classes for ALBERT model."""
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+                        
 from .tokenization_utils import PreTrainedTokenizer
 import logging
 import unicodedata

From c4403006b8301a24bfeec99c39ce8d5d47df570f Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 31 Oct 2019 15:30:11 +0000
Subject: [PATCH 218/269] External MLM head

---
 transformers/configuration_albert.py | 17 ++++++++++++++
 transformers/modeling_albert.py      | 35 +++++++++++++++++++---------
 2 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py
index 04f9fa8d60..b72bbb971e 100644
--- a/transformers/configuration_albert.py
+++ b/transformers/configuration_albert.py
@@ -1,3 +1,20 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" ALBERT model configuration """
+
 from .configuration_utils import PretrainedConfig
 
 class AlbertConfig(PretrainedConfig):
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index b6d1291725..487455e561 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -401,6 +401,26 @@ class AlbertModel(BertModel):
         outputs = (sequence_output, pooled_output) + encoder_outputs[1:]  # add hidden_states and attentions if they are here
         return outputs
 
+class AlbertMLMHead(nn.Module):
+    def __init__(self, config):
+        super(AlbertMLMHead, self).__init__()
+
+        self.LayerNorm = nn.LayerNorm(config.embedding_size)
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+        self.dense = nn.Linear(config.hidden_size, config.embedding_size)
+        self.decoder = nn.Linear(config.embedding_size, config.vocab_size)
+        self.activation = ACT2FN[config.hidden_act]
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
+        hidden_states = self.decoder(hidden_states)
+
+        prediction_scores = hidden_states + self.bias
+
+        return prediction_scores
+
 
 @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
 class AlbertForMaskedLM(BertPreTrainedModel):
@@ -433,13 +453,8 @@ class AlbertForMaskedLM(BertPreTrainedModel):
     def __init__(self, config):
         super(AlbertForMaskedLM, self).__init__(config)
 
-        self.config = config
         self.albert = AlbertModel(config)
-        self.LayerNorm = nn.LayerNorm(config.embedding_size)
-        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
-        self.dense = nn.Linear(config.hidden_size, config.embedding_size)
-        self.decoder = nn.Linear(config.embedding_size, config.vocab_size)
-        self.activation = ACT2FN[config.hidden_act]
+        self.predictions = AlbertMLMHead(config)
 
         self.init_weights()
         self.tie_weights()
@@ -448,17 +463,15 @@ class AlbertForMaskedLM(BertPreTrainedModel):
         """ Make sure we are sharing the input and output embeddings.
             Export to TorchScript can't handle parameter sharing so we are cloning them instead.
         """
-        self._tie_or_clone_weights(self.decoder,
+        self._tie_or_clone_weights(self.predictions.decoder,
                                    self.albert.embeddings.word_embeddings)
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None):
         outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)
         sequence_outputs = outputs[0]
-        hidden_states = self.dense(sequence_outputs)
-        hidden_states = self.activation(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states)
-        prediction_scores = self.decoder(hidden_states)
+
+        prediction_scores = self.predictions(sequence_outputs)
 
         outputs = (prediction_scores,) + outputs[2:]  # Add hidden states and attention if they are here
         if masked_lm_labels is not None:

From 4f3a54bfc8fa8749f6d5b29f110148738a646fcd Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 31 Oct 2019 16:37:34 +0000
Subject: [PATCH 219/269] ALBERT can load pre-trained models. Doesn't inherit
 from BERT anymore.

---
 transformers/__init__.py             |  2 +-
 transformers/configuration_albert.py |  9 ++++++
 transformers/modeling_albert.py      | 44 +++++++++++++++++++++++-----
 transformers/tokenization_albert.py  | 25 +++++++++++++---
 4 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 152d520e7b..bdfb1a0922 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -107,7 +107,7 @@ if is_torch_available():
                                 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
-    from .modeling_albert import (AlbertModel, AlbertForMaskedLM)
+    from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
 
     # Optimization
     from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup,
diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py
index b72bbb971e..c35426768f 100644
--- a/transformers/configuration_albert.py
+++ b/transformers/configuration_albert.py
@@ -17,12 +17,21 @@
 
 from .configuration_utils import PretrainedConfig
 
+ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json",
+    'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json",
+    'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json",
+    'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json",
+}
+
 class AlbertConfig(PretrainedConfig):
     """Configuration for `AlbertModel`.
 
     The default settings match the configuration of model `albert_xxlarge`.
     """
 
+    pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+
     def __init__(self,
                  vocab_size_or_config_json_file=30000,
                  embedding_size=128,
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 487455e561..4da10ed1cb 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -21,6 +21,7 @@ import logging
 import torch
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss
+from transformers.modeling_utils import PreTrainedModel
 from transformers.configuration_albert import AlbertConfig
 from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN
 from .file_utils import add_start_docstrings
@@ -274,6 +275,29 @@ class AlbertTransformer(nn.Module):
         return outputs  # last-layer hidden state, (all hidden states), (all attentions)
 
 
+
+class AlbertPreTrainedModel(PreTrainedModel):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+    config_class = AlbertConfig
+    pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "albert"
+
+    def _init_weights(self, module):
+        """ Initialize the weights.
+        """
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if isinstance(module, (nn.Linear)) and module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
 ALBERT_START_DOCSTRING = r"""    The ALBERT model was proposed in
     `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_
     by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents
@@ -338,7 +362,7 @@ ALBERT_INPUTS_DOCSTRING = r"""
 
 @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.",
                       ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
-class AlbertModel(BertModel):
+class AlbertModel(AlbertPreTrainedModel):
     r"""
     Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
         **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
@@ -358,6 +382,12 @@ class AlbertModel(BertModel):
             list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
             Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
     """
+
+    config_class = AlbertConfig
+    pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    load_tf_weights = load_tf_weights_in_albert
+    base_model_prefix = "albert"
+
     def __init__(self, config):
         super(AlbertModel, self).__init__(config)
 
@@ -369,6 +399,11 @@ class AlbertModel(BertModel):
 
         self.init_weights()
 
+    def _resize_token_embeddings(self, new_num_tokens):
+        old_embeddings = self.embeddings.word_embeddings
+        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
+        self.embeddings.word_embeddings = new_embeddings
+        return self.embeddings.word_embeddings
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
@@ -423,7 +458,7 @@ class AlbertMLMHead(nn.Module):
 
 
 @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
-class AlbertForMaskedLM(BertPreTrainedModel):
+class AlbertForMaskedLM(AlbertPreTrainedModel):
     r"""
         **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
             Labels for computing the masked language modeling loss.
@@ -445,11 +480,6 @@ class AlbertForMaskedLM(BertPreTrainedModel):
             Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
     """
 
-    config_class = AlbertConfig
-    pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
-    load_tf_weights = load_tf_weights_in_albert
-    base_model_prefix = "albert"
-
     def __init__(self, config):
         super(AlbertForMaskedLM, self).__init__(config)
 
diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py
index 7cba99b9e4..acf67c1154 100644
--- a/transformers/tokenization_albert.py
+++ b/transformers/tokenization_albert.py
@@ -15,7 +15,7 @@
 """ Tokenization classes for ALBERT model."""
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
-                        
+
 from .tokenization_utils import PreTrainedTokenizer
 import logging
 import unicodedata
@@ -24,8 +24,25 @@ import os
 from shutil import copyfile
 
 logger = logging.getLogger(__name__)
-
 VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    'vocab_file':
+    {
+    'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model",
+    'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model",
+    'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model",
+    'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model",
+    }
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    'albert-base': 512,
+    'albert-large': 512,
+    'albert-xlarge': 512,
+    'albert-xxlarge': 512,
+}
+
 SPIECE_UNDERLINE = u'▁'
 
 class AlbertTokenizer(PreTrainedTokenizer):
@@ -35,8 +52,8 @@ class AlbertTokenizer(PreTrainedTokenizer):
             - requires `SentencePiece <https://github.com/google/sentencepiece>`_
     """
     vocab_files_names = VOCAB_FILES_NAMES
-    # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
-    # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, vocab_file,
                  do_lower_case=True, remove_space=True, keep_accents=False,

From c9875455929a63f12b81e2dcc8fe30f72137c06b Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 31 Oct 2019 18:48:02 +0000
Subject: [PATCH 220/269] Converting script

---
 transformers/__init__.py                      |  2 +-
 ...lbert_original_tf_checkpoint_to_pytorch.py | 36 +++++++++++++++----
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index bdfb1a0922..db98d5fd44 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -107,7 +107,7 @@ if is_torch_available():
                                 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
-    from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+    from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
 
     # Optimization
     from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup,
diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
index 04877d41b9..5bbaab8c21 100644
--- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
+++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
@@ -1,18 +1,39 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convert ALBERT checkpoint."""
 
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
 
+import argparse
+import torch
 
-from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert
-
+from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_albert
 
+import logging
+logging.basicConfig(level=logging.INFO)
 
 def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path):
     # Initialise PyTorch model
-    config = BertConfig.from_json_file(bert_config_file)
+    config = AlbertConfig.from_json_file(bert_config_file)
     print("Building PyTorch model from configuration: {}".format(str(config)))
-    model = BertForPreTraining(config)
+    model = AlbertForMaskedLM(config)
 
     # Load weights from tf checkpoint
-    load_tf_weights_in_bert(model, config, tf_checkpoint_path)
+    load_tf_weights_in_albert(model, config, tf_checkpoint_path)
 
     # Save pytorch-model
     print("Save PyTorch model to {}".format(pytorch_dump_path))
@@ -31,7 +52,7 @@ if __name__ == "__main__":
                         default = None,
                         type = str,
                         required = True,
-                        help = "The config json file corresponding to the pre-trained BERT model. \n"
+                        help = "The config json file corresponding to the pre-trained ALBERT model. \n"
                             "This specifies the model architecture.")
     parser.add_argument("--pytorch_dump_path",
                         default = None,
@@ -40,5 +61,6 @@ if __name__ == "__main__":
                         help = "Path to the output PyTorch model.")
     args = parser.parse_args()
     convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path,
-                                     args.bert_config_file,
+                                     args.albert_config_file,
                                      args.pytorch_dump_path)
+ 
\ No newline at end of file

From 0d07a23c04c9837234cda402b32246d7581e5bc4 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Fri, 1 Nov 2019 15:07:01 +0000
Subject: [PATCH 221/269] LAMB implementation

---
 transformers/optimization.py | 93 ++++++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/transformers/optimization.py b/transformers/optimization.py
index 99e6cc75e4..90f3dbca9b 100644
--- a/transformers/optimization.py
+++ b/transformers/optimization.py
@@ -167,3 +167,96 @@ class AdamW(Optimizer):
                     p.data.add_(-group['lr'] * group['weight_decay'], p.data)
 
         return loss
+
+
+
+class Lamb(Optimizer):
+    """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training).
+
+    Adapted from the huggingface/transformers ADAM optimizer
+    Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py
+    Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py
+
+
+    Parameters:
+        lr (float): learning rate. Default 1e-3.
+        betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999)
+        eps (float): Adams epsilon. Default: 1e-6
+        weight_decay (float): Weight decay. Default: 0.0
+    """
+    
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True):
+        if lr < 0.0:
+            raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0]))
+        if not 0.0 <= betas[1]  < 1.0:
+            raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1]))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps))
+        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay,
+                        correct_bias=correct_bias)
+        super(Lamb, self).__init__(params, defaults)
+
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            loss = closure()
+
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                grad = p.grad.data
+                if grad.is_sparse:
+                    raise RuntimeError('LAMB does not support sparse gradients.')
+
+                state = self.state[p]
+
+                # State initialization
+                if len(state) == 0:
+                    state['step'] = 0
+                    # Exponential moving average of gradient values
+                    state['exp_avg'] = torch.zeros_like(p.data)
+                    # Exponential moving average of squared gradient values
+                    state['exp_avg_sq'] = torch.zeros_like(p.data)
+
+                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
+                beta1, beta2 = group['betas']
+
+                state['step'] += 1
+
+                # Decay the first and second moment running average coefficient
+                # In-place operations to update the averages at the same time
+                exp_avg.mul_(beta1).add_(1.0 - beta1, grad)
+                exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad)
+                denom = exp_avg_sq.sqrt().add_(group['eps'])
+
+
+                # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py
+                step_size = group['lr']
+                weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10)
+
+                adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps'])
+                if group['weight_decay'] != 0:
+                    adam_step.add_(group['weight_decay'], p.data)
+
+                adam_norm = adam_step.pow(2).sum().sqrt()
+                if weight_norm == 0 or adam_norm == 0:
+                    trust_ratio = 1
+                else:
+                    trust_ratio = weight_norm / adam_norm
+
+                
+                state['weight_norm'] = weight_norm
+                state['adam_norm'] = adam_norm
+                state['trust_ratio'] = trust_ratio
+
+                p.data.add_(-step_size * trust_ratio, adam_step)
+        return loss

From 6637a77f807615ef2427c0390a015f4eb4814fb4 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Fri, 1 Nov 2019 15:17:31 +0000
Subject: [PATCH 222/269] AlbertForSequenceClassification

---
 transformers/__init__.py        |  3 +-
 transformers/modeling_albert.py | 77 ++++++++++++++++++++++++++++++++-
 2 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index db98d5fd44..51995942ce 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -107,7 +107,8 @@ if is_torch_available():
                                 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
-    from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+    from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification,
+                                load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
 
     # Optimization
     from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup,
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 4da10ed1cb..bba6767079 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -20,10 +20,10 @@ import math
 import logging
 import torch
 import torch.nn as nn
-from torch.nn import CrossEntropyLoss
+from torch.nn import CrossEntropyLoss, MSELoss
 from transformers.modeling_utils import PreTrainedModel
 from transformers.configuration_albert import AlbertConfig
-from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN
+from transformers.modeling_bert import BertEmbeddings, BertSelfAttention, prune_linear_layer, ACT2FN
 from .file_utils import add_start_docstrings
 
 logger = logging.getLogger(__name__)
@@ -510,3 +510,76 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
             outputs = (masked_lm_loss,) + outputs
 
         return outputs
+
+
+@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of
+    the pooled output) e.g. for GLUE tasks. """,
+    ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
+class AlbertForSequenceClassification(AlbertPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base')
+        model = AlbertForSequenceClassification.from_pretrained('albert-base')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+
+    """
+    def __init__(self, config):
+        super(AlbertForSequenceClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.albert = AlbertModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, labels=None):
+
+        outputs = self.albert(input_ids,
+                            attention_mask=attention_mask,
+                            token_type_ids=token_type_ids,
+                            position_ids=position_ids, 
+                            head_mask=head_mask)
+
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        if labels is not None:
+            if self.num_labels == 1:
+                #  We are doing regression
+                loss_fct = MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions)
\ No newline at end of file

From c110c41fdb3b363528336ceda3b9fb46f026ad9c Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Fri, 1 Nov 2019 21:59:40 +0000
Subject: [PATCH 223/269] Run GLUE and remove LAMB

---
 examples/run_glue.py         | 14 ++++--
 transformers/optimization.py | 93 ------------------------------------
 2 files changed, 10 insertions(+), 97 deletions(-)

diff --git a/examples/run_glue.py b/examples/run_glue.py
index 527e440075..550a0b8175 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -47,7 +47,11 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   XLNetTokenizer,
                                   DistilBertConfig,
                                   DistilBertForSequenceClassification,
-                                  DistilBertTokenizer)
+                                  DistilBertTokenizer,
+                                  AlbertConfig,
+                                  AlbertForSequenceClassification, 
+                                  AlbertTokenizer,
+                                )
 
 from transformers import AdamW, get_linear_schedule_with_warmup
 
@@ -66,7 +70,8 @@ MODEL_CLASSES = {
     'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
     'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
     'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
-    'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
+    'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer),
+    'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer)
 }
 
 
@@ -99,6 +104,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
+
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
     scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
@@ -317,7 +323,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
         all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
     elif output_mode == "regression":
         all_labels = torch.tensor([f.label for f in features], dtype=torch.float)
-
+ 
     dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
     return dataset
 
@@ -361,7 +367,7 @@ def main():
     parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int,
                         help="Batch size per GPU/CPU for evaluation.")
     parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
-                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")     
     parser.add_argument("--learning_rate", default=5e-5, type=float,
                         help="The initial learning rate for Adam.")
     parser.add_argument("--weight_decay", default=0.0, type=float,
diff --git a/transformers/optimization.py b/transformers/optimization.py
index 90f3dbca9b..99e6cc75e4 100644
--- a/transformers/optimization.py
+++ b/transformers/optimization.py
@@ -167,96 +167,3 @@ class AdamW(Optimizer):
                     p.data.add_(-group['lr'] * group['weight_decay'], p.data)
 
         return loss
-
-
-
-class Lamb(Optimizer):
-    """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training).
-
-    Adapted from the huggingface/transformers ADAM optimizer
-    Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py
-    Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py
-
-
-    Parameters:
-        lr (float): learning rate. Default 1e-3.
-        betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999)
-        eps (float): Adams epsilon. Default: 1e-6
-        weight_decay (float): Weight decay. Default: 0.0
-    """
-    
-    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True):
-        if lr < 0.0:
-            raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr))
-        if not 0.0 <= betas[0] < 1.0:
-            raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0]))
-        if not 0.0 <= betas[1]  < 1.0:
-            raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1]))
-        if not 0.0 <= eps:
-            raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps))
-        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay,
-                        correct_bias=correct_bias)
-        super(Lamb, self).__init__(params, defaults)
-
-    def step(self, closure=None):
-        """Performs a single optimization step.
-
-        Arguments:
-            closure (callable, optional): A closure that reevaluates the model
-                and returns the loss.
-        """
-        loss = None
-        if closure is not None:
-            loss = closure()
-
-        for group in self.param_groups:
-            for p in group['params']:
-                if p.grad is None:
-                    continue
-                grad = p.grad.data
-                if grad.is_sparse:
-                    raise RuntimeError('LAMB does not support sparse gradients.')
-
-                state = self.state[p]
-
-                # State initialization
-                if len(state) == 0:
-                    state['step'] = 0
-                    # Exponential moving average of gradient values
-                    state['exp_avg'] = torch.zeros_like(p.data)
-                    # Exponential moving average of squared gradient values
-                    state['exp_avg_sq'] = torch.zeros_like(p.data)
-
-                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
-                beta1, beta2 = group['betas']
-
-                state['step'] += 1
-
-                # Decay the first and second moment running average coefficient
-                # In-place operations to update the averages at the same time
-                exp_avg.mul_(beta1).add_(1.0 - beta1, grad)
-                exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad)
-                denom = exp_avg_sq.sqrt().add_(group['eps'])
-
-
-                # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py
-                step_size = group['lr']
-                weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10)
-
-                adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps'])
-                if group['weight_decay'] != 0:
-                    adam_step.add_(group['weight_decay'], p.data)
-
-                adam_norm = adam_step.pow(2).sum().sqrt()
-                if weight_norm == 0 or adam_norm == 0:
-                    trust_ratio = 1
-                else:
-                    trust_ratio = weight_norm / adam_norm
-
-                
-                state['weight_norm'] = weight_norm
-                state['adam_norm'] = adam_norm
-                state['trust_ratio'] = trust_ratio
-
-                p.data.add_(-step_size * trust_ratio, adam_step)
-        return loss

From 70d99980ded1565c9e8efa2cd04c21572b664f2f Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Mon, 4 Nov 2019 11:34:30 -0500
Subject: [PATCH 224/269] ALBERT-V2

---
 transformers/configuration_albert.py          | 12 ++++++----
 ...lbert_original_tf_checkpoint_to_pytorch.py |  5 ++--
 transformers/modeling_albert.py               | 16 ++++++++-----
 transformers/tokenization_albert.py           | 24 ++++++++++++-------
 4 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py
index c35426768f..de665c9b1c 100644
--- a/transformers/configuration_albert.py
+++ b/transformers/configuration_albert.py
@@ -18,10 +18,14 @@
 from .configuration_utils import PretrainedConfig
 
 ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json",
-    'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json",
-    'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json",
-    'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json",
+    'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json",
+    'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json",
+    'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json",
+    'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json",
+    'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json",
+    'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-config.json",
+    'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-config.json",
+    'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-config.json",
 }
 
 class AlbertConfig(PretrainedConfig):
diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
index 5bbaab8c21..b6476b4fb6 100644
--- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
+++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py
@@ -26,9 +26,10 @@ from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_alb
 import logging
 logging.basicConfig(level=logging.INFO)
 
-def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path):
+
+def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_file, pytorch_dump_path):
     # Initialise PyTorch model
-    config = AlbertConfig.from_json_file(bert_config_file)
+    config = AlbertConfig.from_json_file(albert_config_file)
     print("Building PyTorch model from configuration: {}".format(str(config)))
     model = AlbertForMaskedLM(config)
 
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index bba6767079..51cb0a6d23 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -30,10 +30,14 @@ logger = logging.getLogger(__name__)
 
 
 ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin",
-    'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin",
-    'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin",
-    'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin",
+    'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin",
+    'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin",
+    'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin",
+    'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin",
+    'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin",
+    'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-pytorch_model.bin",
+    'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-pytorch_model.bin",
+    'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-pytorch_model.bin",
 }
 
 
@@ -538,8 +542,8 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
 
     Examples::
 
-        tokenizer = AlbertTokenizer.from_pretrained('albert-base')
-        model = AlbertForSequenceClassification.from_pretrained('albert-base')
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForSequenceClassification.from_pretrained('albert-base-v2')
         input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
         labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
         outputs = model(input_ids, labels=labels)
diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py
index acf67c1154..2f9af0b0bc 100644
--- a/transformers/tokenization_albert.py
+++ b/transformers/tokenization_albert.py
@@ -29,18 +29,26 @@ VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'}
 PRETRAINED_VOCAB_FILES_MAP = {
     'vocab_file':
     {
-    'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model",
-    'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model",
-    'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model",
-    'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model",
+        'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model",
+        'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model",
+        'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model",
+        'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model",
+        'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model",
+        'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-spiece.model",
+        'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-spiece.model",
+        'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-spiece.model",
     }
 }
 
 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    'albert-base': 512,
-    'albert-large': 512,
-    'albert-xlarge': 512,
-    'albert-xxlarge': 512,
+    'albert-base-v1': 512,
+    'albert-large-v1': 512,
+    'albert-xlarge-v1': 512,
+    'albert-xxlarge-v1': 512,
+    'albert-base-v2': 512,
+    'albert-large-v2': 512,
+    'albert-xlarge-v2': 512,
+    'albert-xxlarge-v2': 512,
 }
 
 SPIECE_UNDERLINE = u'▁'

From 4374eaea786399a948b4b34d39fc614ded5e1de6 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 6 Nov 2019 20:47:42 +0000
Subject: [PATCH 225/269] ALBERT for SQuAD

---
 examples/run_squad.py           | 12 +++--
 transformers/__init__.py        |  1 +
 transformers/modeling_albert.py | 93 ++++++++++++++++++++++++++++++++-
 3 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/examples/run_squad.py b/examples/run_squad.py
index 69088d73c3..59683c0668 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -43,7 +43,8 @@ from transformers import (WEIGHTS_NAME, BertConfig,
                                   XLMTokenizer, XLNetConfig,
                                   XLNetForQuestionAnswering,
                                   XLNetTokenizer,
-                                  DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
+                                  DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer,
+                                  AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer)
 
 from transformers import AdamW, get_linear_schedule_with_warmup
 
@@ -65,7 +66,8 @@ MODEL_CLASSES = {
     'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer),
     'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
     'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
-    'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)
+    'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
+    'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer)
 }
 
 def set_seed(args):
@@ -128,7 +130,7 @@ def train(args, train_dataset, model, tokenizer):
     logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
     logger.info("  Total optimization steps = %d", t_total)
 
-    global_step = 0
+    global_step = 1
     tr_loss, logging_loss = 0.0, 0.0
     model.zero_grad()
     train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
@@ -537,7 +539,7 @@ def main():
         torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
 
         # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
+        model = model_class.from_pretrained(args.output_dir, force_download=True)
         tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
         model.to(args.device)
 
@@ -555,7 +557,7 @@ def main():
         for checkpoint in checkpoints:
             # Reload the model
             global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
-            model = model_class.from_pretrained(checkpoint)
+            model = model_class.from_pretrained(checkpoint, force_download=True)
             model.to(args.device)
 
             # Evaluate
diff --git a/transformers/__init__.py b/transformers/__init__.py
index 51995942ce..81e659329d 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -108,6 +108,7 @@ if is_torch_available():
     from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model
 
     from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification,
+                                AlbertForQuestionAnswering,
                                 load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
 
     # Optimization
diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 51cb0a6d23..2540218e69 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -586,4 +586,95 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
                 loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
             outputs = (loss,) + outputs
 
-        return outputs  # (loss), logits, (hidden_states), (attentions)
\ No newline at end of file
+        return outputs  # (loss), logits, (hidden_states), (attentions)
+
+
+
+@add_start_docstrings("""Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of
+    the hidden-states output to compute `span start logits` and `span end logits`). """,
+    ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
+class AlbertForQuestionAnswering(AlbertPreTrainedModel):
+    r"""
+        **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for position (index) of the start of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`).
+            Position outside of the sequence are not taken into account for computing the loss.
+        **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+            Labels for position (index) of the end of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`).
+            Position outside of the sequence are not taken into account for computing the loss.
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
+        **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
+            Span-start scores (before SoftMax).
+        **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
+            Span-end scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForQuestionAnswering.from_pretrained('albert-base-v2')
+        question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+        input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]"
+        input_ids = tokenizer.encode(input_text)
+        token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] 
+        start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))
+        all_tokens = tokenizer.convert_ids_to_tokens(input_ids)  
+        print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1]))
+        # a nice puppet
+
+
+    """
+    def __init__(self, config):
+        super(AlbertForQuestionAnswering, self).__init__(config)
+        self.num_labels = config.num_labels
+
+        self.albert = AlbertModel(config)
+        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
+
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                start_positions=None, end_positions=None):
+
+        outputs = self.albert(input_ids,
+                            attention_mask=attention_mask,
+                            token_type_ids=token_type_ids,
+                            position_ids=position_ids, 
+                            head_mask=head_mask)
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1)
+        end_logits = end_logits.squeeze(-1)
+
+        outputs = (start_logits, end_logits,) + outputs[2:]
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions.clamp_(0, ignored_index)
+            end_positions.clamp_(0, ignored_index)
+
+            loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+            outputs = (total_loss,) + outputs
+
+        return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)

From abb23a78bab17ec09dde4635c32f4aa21c15fa83 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 7 Nov 2019 17:09:16 +0000
Subject: [PATCH 226/269] Head pruning for ALBERT

---
 transformers/modeling_albert.py            | 42 ++++++++++++++++++++++
 transformers/tests/modeling_albert_test.py | 12 ++++---
 2 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 2540218e69..89ece4b61e 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -145,6 +145,29 @@ class AlbertAttention(BertSelfAttention):
         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.pruned_heads = set()
 
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        mask = torch.ones(self.num_attention_heads, self.attention_head_size)
+        heads = set(heads) - self.pruned_heads  # Convert to set and emove already pruned heads
+        for head in heads:
+            # Compute how many pruned heads are before the head and move the index accordingly
+            head = head - sum(1 if h < head else 0 for h in self.pruned_heads)
+            mask[head] = 0
+        mask = mask.view(-1).contiguous().eq(1)
+        index = torch.arange(len(mask))[mask].long()
+
+        # Prune linear layers
+        self.query = prune_linear_layer(self.query, index)
+        self.key = prune_linear_layer(self.key, index)
+        self.value = prune_linear_layer(self.value, index)
+        self.dense = prune_linear_layer(self.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.num_attention_heads = self.num_attention_heads - len(heads)
+        self.all_head_size = self.attention_head_size * self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
+
     def forward(self, input_ids, attention_mask=None, head_mask=None):
         mixed_query_layer = self.query(input_ids)
         mixed_key_layer = self.key(input_ids)
@@ -409,6 +432,25 @@ class AlbertModel(AlbertPreTrainedModel):
         self.embeddings.word_embeddings = new_embeddings
         return self.embeddings.word_embeddings
 
+    def _prune_heads(self, heads_to_prune):
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            ALBERT has a different architecture in that its layers are shared across groups, which then has inner groups.
+            If an ALBERT model has 12 hidden layers and 2 hidden groups, with two inner groups, there
+            is a total of 4 different layers.
+
+            These layers are flattened: the indices [0,1] correspond to the two inner groups of the first hidden layer,
+            while [2,3] correspond to the two inner groups of the second hidden layer.
+
+            Any layer with in index other than [0,1,2,3] will result in an error.
+            See base class PreTrainedModel for more information about head pruning
+        """
+        for layer, heads in heads_to_prune.items():
+            group_idx = int(layer / self.config.inner_group_num)
+            inner_group_idx = int(layer - group_idx * self.config.inner_group_num)
+            self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads)
+
+
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
             attention_mask = torch.ones_like(input_ids)
diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py
index 46a2eeb729..979e0488eb 100644
--- a/transformers/tests/modeling_albert_test.py
+++ b/transformers/tests/modeling_albert_test.py
@@ -35,7 +35,6 @@ else:
 class AlbertModelTest(CommonTestCases.CommonModelTester):
 
     all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else ()
-    test_pruning = False
     test_head_masking = False
 
     class AlbertModelTester(object):
@@ -49,9 +48,10 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
                      use_token_type_ids=True,
                      use_labels=True,
                      vocab_size=99,
-                     hidden_size=32,
-                     num_hidden_layers=5,
-                     num_attention_heads=4,
+                     hidden_size=36,
+                     num_hidden_layers=6,
+                     num_hidden_groups=6,
+                     num_attention_heads=6,
                      intermediate_size=37,
                      hidden_act="gelu",
                      hidden_dropout_prob=0.1,
@@ -86,6 +86,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
             self.num_labels = num_labels
             self.num_choices = num_choices
             self.scope = scope
+            self.num_hidden_groups = num_hidden_groups
 
         def prepare_config_and_inputs(self):
             input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
@@ -117,7 +118,8 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
                 attention_probs_dropout_prob=self.attention_probs_dropout_prob,
                 max_position_embeddings=self.max_position_embeddings,
                 type_vocab_size=self.type_vocab_size,
-                initializer_range=self.initializer_range)
+                initializer_range=self.initializer_range,
+                num_hidden_groups=self.num_hidden_groups)
 
             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 

From 16263f9685eaf459408f33c9790a967012b93fa5 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 7 Nov 2019 17:29:29 +0000
Subject: [PATCH 227/269] Headmasking

---
 transformers/modeling_albert.py            | 11 ++++++-----
 transformers/tests/modeling_albert_test.py |  1 -
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 89ece4b61e..6682930d89 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -224,7 +224,7 @@ class AlbertLayer(nn.Module):
         self.activation = ACT2FN[config.hidden_act]
 
     def forward(self, hidden_states, attention_mask=None, head_mask=None):
-        attention_output = self.attention(hidden_states, attention_mask)
+        attention_output = self.attention(hidden_states, attention_mask, head_mask)
         ffn_output = self.ffn(attention_output[0])
         ffn_output = self.activation(ffn_output)
         ffn_output = self.ffn_output(ffn_output)
@@ -245,8 +245,8 @@ class AlbertLayerGroup(nn.Module):
         layer_hidden_states = ()
         layer_attentions = ()
 
-        for albert_layer in self.albert_layers:
-            layer_output = albert_layer(hidden_states, attention_mask, head_mask)
+        for layer_index, albert_layer in enumerate(self.albert_layers):
+            layer_output = albert_layer(hidden_states, attention_mask, head_mask[layer_index])
             hidden_states = layer_output[0]
 
             if self.output_attentions:
@@ -283,7 +283,8 @@ class AlbertTransformer(nn.Module):
 
         for layer_idx in range(self.config.num_hidden_layers):
             group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups)
-            layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask)  
+            layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups)
+            layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group])  
         
             hidden_states = layer_group_output[0]
 
@@ -544,7 +545,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
 
     def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
                 masked_lm_labels=None):
-        outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)
+        outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask)
         sequence_outputs = outputs[0]
 
         prediction_scores = self.predictions(sequence_outputs)
diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py
index 979e0488eb..466f473332 100644
--- a/transformers/tests/modeling_albert_test.py
+++ b/transformers/tests/modeling_albert_test.py
@@ -35,7 +35,6 @@ else:
 class AlbertModelTest(CommonTestCases.CommonModelTester):
 
     all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else ()
-    test_head_masking = False
 
     class AlbertModelTester(object):
 

From 9d5c49546fedb3d52f76f97f4043a07e08ded918 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 7 Nov 2019 17:32:52 +0000
Subject: [PATCH 228/269] Tests for AlbertForQuestionAnswering
 AlbertForSequenceClassification

---
 transformers/tests/modeling_albert_test.py | 45 +++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py
index 466f473332..da87709df1 100644
--- a/transformers/tests/modeling_albert_test.py
+++ b/transformers/tests/modeling_albert_test.py
@@ -26,7 +26,9 @@ from .modeling_common_test import (CommonTestCases, ids_tensor)
 from .configuration_common_test import ConfigTester
 
 if is_torch_available():
-    from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM)
+    from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM,
+                              AlbertForSequenceClassification, AlbertForQuestionAnswering,
+                              )
     from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -157,6 +159,39 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.vocab_size])
             self.check_loss_output(result)
 
+        def create_and_check_albert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = AlbertForQuestionAnswering(config=config)
+            model.eval()
+            loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids,
+                                                   start_positions=sequence_labels, end_positions=sequence_labels)
+            result = {
+                "loss": loss,
+                "start_logits": start_logits,
+                "end_logits": end_logits,
+            }
+            self.parent.assertListEqual(
+                list(result["start_logits"].size()),
+                [self.batch_size, self.seq_length])
+            self.parent.assertListEqual(
+                list(result["end_logits"].size()),
+                [self.batch_size, self.seq_length])
+            self.check_loss_output(result)
+
+
+        def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = AlbertForSequenceClassification(config)
+            model.eval()
+            loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.num_labels])
+            self.check_loss_output(result)
+
 
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
@@ -180,6 +215,14 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs)
 
+    def test_for_question_answering(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_for_question_answering(*config_and_inputs)
+
+    def test_for_sequence_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_for_sequence_classification(*config_and_inputs)
+
     @pytest.mark.slow
     def test_model_from_pretrained(self):
         cache_dir = "/tmp/transformers_test/"

From d9daad98c744e115bfb425f316a5e7d4f405a9a5 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 7 Nov 2019 19:55:43 +0000
Subject: [PATCH 229/269] Re-ordering of group_idx/layer_idx + Python 2 tests

---
 transformers/modeling_albert.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 6682930d89..640af537d0 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -281,11 +281,17 @@ class AlbertTransformer(nn.Module):
         if self.output_hidden_states:
             all_hidden_states = (hidden_states,)
 
-        for layer_idx in range(self.config.num_hidden_layers):
-            group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups)
+        for i in range(self.config.num_hidden_layers):
+            # Number of layers in a hidden group
             layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups)
+
+            # Index of the hidden group
+            group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups))
+
+            # Index of the layer inside the group
+            layer_idx = int(i - group_idx * layers_per_group)
+            
             layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group])  
-        
             hidden_states = layer_group_output[0]
 
             if self.output_attentions:

From f6f382532bf40a5869dc14e3eefa451646c19ded Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 7 Nov 2019 23:40:45 +0000
Subject: [PATCH 230/269] ALBERT in TF2

---
 transformers/__init__.py                      |   6 +-
 .../convert_pytorch_checkpoint_to_tf2.py      |  13 +-
 transformers/modeling_tf_albert.py            | 723 ++++++++++++++++++
 3 files changed, 736 insertions(+), 6 deletions(-)
 create mode 100644 transformers/modeling_tf_albert.py

diff --git a/transformers/__init__.py b/transformers/__init__.py
index 81e659329d..a409ef772e 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -58,8 +58,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
-from .configuration_albert import AlbertConfig, ALBERT
-from .configuration_albert import AlbertConfig
+from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
 
 # Modeling
@@ -169,6 +168,9 @@ if is_tf_available():
                                     TFCTRLLMHeadModel,
                                     TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP)
 
+    from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM,
+                                    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+
 # TF 2.0 <=> PyTorch conversion utilities
 from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name,
                                         load_pytorch_checkpoint_in_tf2_model,
diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py
index e673b77dcc..d1776e9c14 100644
--- a/transformers/convert_pytorch_checkpoint_to_tf2.py
+++ b/transformers/convert_pytorch_checkpoint_to_tf2.py
@@ -33,7 +33,8 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model,
                                   OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP,
                                   RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
                                   DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
-                                  CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP)
+                                  CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP,
+                                  AlbertConfig, TFAlbertForMaskedLM, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP)
 
 if is_torch_available():
     import torch
@@ -46,7 +47,8 @@ if is_torch_available():
                                       OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP,
                                       RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP,
                                       DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP,
-                                      CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP)
+                                      CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP,
+                                      AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
 else:
     (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP,
     GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP,
@@ -56,7 +58,8 @@ else:
     OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP,
     RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP,
     DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP,
-    CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) = (
+    CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP,
+    AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) = (
         None, None, None, None,
         None, None,
         None, None,
@@ -65,6 +68,7 @@ else:
         None, None,
         None, None, None,
         None, None, None,
+        None, None,
         None, None)
 
 
@@ -85,7 +89,8 @@ MODEL_CLASSES = {
     'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP),
     'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP),
     'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP),
-    'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP)
+    'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP),
+    'albert': (AlbertConfig, TFAlbertForMaskedLM, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP)
 }
 
 def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True):
diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py
new file mode 100644
index 0000000000..8861b7add8
--- /dev/null
+++ b/transformers/modeling_tf_albert.py
@@ -0,0 +1,723 @@
+# coding=utf-8
+# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" TF 2.0 ALBERT model. """
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import logging
+import math
+import os
+import sys
+from io import open
+
+import numpy as np
+import tensorflow as tf
+
+from .configuration_albert import AlbertConfig
+from .modeling_tf_utils import TFPreTrainedModel, get_initializer
+from .modeling_tf_bert import ACT2FN, TFBertSelfAttention
+from .file_utils import add_start_docstrings
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
+    # TODO FILL THAT UP
+}
+
+
+class TFAlbertEmbeddings(tf.keras.layers.Layer):
+    """Construct the embeddings from word, position and token_type embeddings.
+    """
+
+    def __init__(self, config, **kwargs):
+        super(TFAlbertEmbeddings, self).__init__(**kwargs)
+
+        self.config = config
+        self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings,
+                                                             config.embedding_size,
+                                                             embeddings_initializer=get_initializer(
+                                                                 self.config.initializer_range),
+                                                             name='position_embeddings')
+        self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size,
+                                                               config.embedding_size,
+                                                               embeddings_initializer=get_initializer(
+                                                                   self.config.initializer_range),
+                                                               name='token_type_embeddings')
+
+        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
+        # any TensorFlow checkpoint file
+        self.LayerNorm = tf.keras.layers.LayerNormalization(
+            epsilon=config.layer_norm_eps, name='LayerNorm')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+
+    def build(self, input_shape):
+        """Build shared word embedding layer """
+        with tf.name_scope("word_embeddings"):
+            # Create and initialize weights. The random normal initializer was chosen
+            # arbitrarily, and works well.
+            self.word_embeddings = self.add_weight(
+                "weight",
+                shape=[self.config.vocab_size, self.config.embedding_size],
+                initializer=get_initializer(self.config.initializer_range))
+        super(TFAlbertEmbeddings, self).build(input_shape)
+
+    def call(self, inputs, mode="embedding", training=False):
+        """Get token embeddings of inputs.
+        Args:
+            inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids)
+            mode: string, a valid value is one of "embedding" and "linear".
+        Returns:
+            outputs: (1) If mode == "embedding", output embedding tensor, float32 with
+                shape [batch_size, length, embedding_size]; (2) mode == "linear", output
+                linear tensor, float32 with shape [batch_size, length, vocab_size].
+        Raises:
+            ValueError: if mode is not valid.
+
+        Shared weights logic adapted from
+            https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
+        """
+        if mode == "embedding":
+            return self._embedding(inputs, training=training)
+        elif mode == "linear":
+            return self._linear(inputs)
+        else:
+            raise ValueError("mode {} is not valid.".format(mode))
+
+    def _embedding(self, inputs, training=False):
+        """Applies embedding based on inputs tensor."""
+        input_ids, position_ids, token_type_ids = inputs
+
+        seq_length = tf.shape(input_ids)[1]
+        if position_ids is None:
+            position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :]
+        if token_type_ids is None:
+            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+
+        words_embeddings = tf.gather(self.word_embeddings, input_ids)
+        position_embeddings = self.position_embeddings(position_ids)
+        token_type_embeddings = self.token_type_embeddings(token_type_ids)
+
+        embeddings = words_embeddings + position_embeddings + token_type_embeddings
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings, training=training)
+        return embeddings
+
+    def _linear(self, inputs):
+        """Computes logits by running inputs through a linear layer.
+            Args:
+                inputs: A float32 tensor with shape [batch_size, length, embedding_size]
+            Returns:
+                float32 tensor with shape [batch_size, length, vocab_size].
+        """
+        batch_size = tf.shape(inputs)[0]
+        length = tf.shape(inputs)[1]
+
+        print(inputs.shape)
+
+        x = tf.reshape(inputs, [-1, self.config.embedding_size])
+
+        print(x.shape, self.word_embeddings)
+
+        logits = tf.matmul(x, self.word_embeddings, transpose_b=True)
+
+        print([batch_size, length, self.config.vocab_size])
+        return tf.reshape(logits, [batch_size, length, self.config.vocab_size])
+
+
+class TFAlbertSelfAttention(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFAlbertSelfAttention, self).__init__(**kwargs)
+        if config.hidden_size % config.num_attention_heads != 0:
+            raise ValueError(
+                "The hidden size (%d) is not a multiple of the number of attention "
+                "heads (%d)" % (config.hidden_size, config.num_attention_heads))
+        self.output_attentions = config.output_attentions
+
+        self.num_attention_heads = config.num_attention_heads
+        assert config.hidden_size % config.num_attention_heads == 0
+        self.attention_head_size = int(
+            config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = tf.keras.layers.Dense(self.all_head_size,
+                                           kernel_initializer=get_initializer(
+                                               config.initializer_range),
+                                           name='query')
+        self.key = tf.keras.layers.Dense(self.all_head_size,
+                                         kernel_initializer=get_initializer(
+                                             config.initializer_range),
+                                         name='key')
+        self.value = tf.keras.layers.Dense(self.all_head_size,
+                                           kernel_initializer=get_initializer(
+                                               config.initializer_range),
+                                           name='value')
+
+        self.dropout = tf.keras.layers.Dropout(
+            config.attention_probs_dropout_prob)
+
+    def transpose_for_scores(self, x, batch_size):
+        x = tf.reshape(
+            x, (batch_size, -1, self.num_attention_heads, self.attention_head_size))
+        return tf.transpose(x, perm=[0, 2, 1, 3])
+
+    def call(self, inputs, training=False):
+        hidden_states, attention_mask, head_mask = inputs
+
+        batch_size = tf.shape(hidden_states)[0]
+        mixed_query_layer = self.query(hidden_states)
+        mixed_key_layer = self.key(hidden_states)
+        mixed_value_layer = self.value(hidden_states)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer, batch_size)
+        key_layer = self.transpose_for_scores(mixed_key_layer, batch_size)
+        value_layer = self.transpose_for_scores(mixed_value_layer, batch_size)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        # (batch size, num_heads, seq_len_q, seq_len_k)
+        attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True)
+        # scale attention_scores
+        dk = tf.cast(tf.shape(key_layer)[-1], tf.float32)
+        attention_scores = attention_scores / tf.math.sqrt(dk)
+
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in TFAlbertModel call() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = tf.nn.softmax(attention_scores, axis=-1)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs, training=training)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = tf.matmul(attention_probs, value_layer)
+
+        context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3])
+        context_layer = tf.reshape(context_layer,
+                                   (batch_size, -1, self.all_head_size))  # (batch_size, seq_len_q, all_head_size)
+
+        outputs = (context_layer, attention_probs) if self.output_attentions else (
+            context_layer,)
+        return outputs
+
+
+class TFAlbertSelfOutput(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFAlbertSelfOutput, self).__init__(**kwargs)
+        self.dense = tf.keras.layers.Dense(config.hidden_size,
+                                           kernel_initializer=get_initializer(
+                                               config.initializer_range),
+                                           name='dense')
+        self.LayerNorm = tf.keras.layers.LayerNormalization(
+            epsilon=config.layer_norm_eps, name='LayerNorm')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+
+    def call(self, inputs, training=False):
+        hidden_states, input_tensor = inputs
+
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states, training=training)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class TFAlbertAttention(TFBertSelfAttention):
+    def __init__(self, config, **kwargs):
+        super(TFAlbertAttention, self).__init__(config, **kwargs)
+
+        self.hidden_size = config.hidden_size
+        self.dense = tf.keras.layers.Dense(config.hidden_size,
+                                           kernel_initializer=get_initializer(
+                                               config.initializer_range),
+                                           name='dense')
+        self.LayerNorm = tf.keras.layers.LayerNormalization(
+            epsilon=config.layer_norm_eps, name='LayerNorm')
+        self.pruned_heads = set()
+
+    def prune_heads(self, heads):
+        raise NotImplementedError
+
+    def call(self, inputs, training=False):
+        input_tensor, attention_mask, head_mask = inputs
+
+        batch_size = tf.shape(input_tensor)[0]
+        mixed_query_layer = self.query(input_tensor)
+        mixed_key_layer = self.key(input_tensor)
+        mixed_value_layer = self.value(input_tensor)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer, batch_size)
+        key_layer = self.transpose_for_scores(mixed_key_layer, batch_size)
+        value_layer = self.transpose_for_scores(mixed_value_layer, batch_size)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        # (batch size, num_heads, seq_len_q, seq_len_k)
+        attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True)
+        # scale attention_scores
+        dk = tf.cast(tf.shape(key_layer)[-1], tf.float32)
+        attention_scores = attention_scores / tf.math.sqrt(dk)
+
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in TFBertModel call() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = tf.nn.softmax(attention_scores, axis=-1)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs, training=training)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = tf.matmul(attention_probs, value_layer)
+
+        context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3])
+        context_layer = tf.reshape(context_layer,
+                                   (batch_size, -1, self.all_head_size))  # (batch_size, seq_len_q, all_head_size)
+
+        self_outputs = (context_layer, attention_probs) if self.output_attentions else (
+            context_layer,)
+
+        hidden_states = self_outputs[0]
+
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states, training=training)
+        attention_output = self.LayerNorm(hidden_states + input_tensor)
+
+        # add attentions if we output them
+        outputs = (attention_output,) + self_outputs[1:]
+        return outputs
+
+
+class TFAlbertLayer(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFAlbertLayer, self).__init__(**kwargs)
+        self.attention = TFAlbertAttention(config, name='attention')
+
+        self.ffn = tf.keras.layers.Dense(config.intermediate_size, kernel_initializer=get_initializer(
+            config.initializer_range), name='ffn')
+
+        if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)):
+            self.activation = ACT2FN[config.hidden_act]
+        else:
+            self.activation = config.hidden_act
+
+        self.ffn_output = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer(
+            config.initializer_range), name='ffn_output')
+        self.full_layer_layer_norm = tf.keras.layers.LayerNormalization(
+            epsilon=config.layer_norm_eps, name='full_layer_layer_norm')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+
+    def call(self, inputs, training=False):
+        hidden_states, attention_mask, head_mask = inputs
+
+        attention_outputs = self.attention(
+            [hidden_states, attention_mask, head_mask], training=training)
+        ffn_output = self.ffn(attention_outputs[0])
+        ffn_output = self.activation(ffn_output)
+        ffn_output = self.ffn_output(ffn_output)
+
+        hidden_states = self.dropout(hidden_states, training=training)
+        hidden_states = self.full_layer_layer_norm(
+            ffn_output + attention_outputs[0])
+
+        # add attentions if we output them
+        outputs = (hidden_states,) + attention_outputs[1:]
+        return outputs
+
+
+class TFAlbertLayerGroup(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFAlbertLayerGroup, self).__init__(**kwargs)
+
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
+        self.albert_layers = [TFAlbertLayer(config, name="albert_layers_._{}".format(
+            i)) for i in range(config.inner_group_num)]
+
+    def call(self, inputs, training=False):
+        hidden_states, attention_mask, head_mask = inputs
+
+        layer_hidden_states = ()
+        layer_attentions = ()
+
+        for layer_index, albert_layer in enumerate(self.albert_layers):
+            layer_output = albert_layer(
+                [hidden_states, attention_mask, head_mask[layer_index]], training=training)
+            hidden_states = layer_output[0]
+
+            if self.output_attentions:
+                layer_attentions = layer_attentions + (layer_output[1],)
+
+            if self.output_hidden_states:
+                layer_hidden_states = layer_hidden_states + (hidden_states,)
+
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (layer_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (layer_attentions,)
+        # last-layer hidden state, (layer hidden states), (layer attentions)
+        return outputs
+
+
+class TFAlbertTransformer(tf.keras.layers.Layer):
+    def __init__(self, config, **kwargs):
+        super(TFAlbertTransformer, self).__init__(**kwargs)
+
+        self.config = config
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
+        self.embedding_hidden_mapping_in = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer(
+            config.initializer_range), name='embedding_hidden_mapping_in')
+        self.albert_layer_groups = [TFAlbertLayerGroup(
+            config, name="albert_layer_groups_._{}".format(i)) for i in range(config.num_hidden_groups)]
+
+    def call(self, inputs, training=False):
+        hidden_states, attention_mask, head_mask = inputs
+
+        hidden_states = self.embedding_hidden_mapping_in(hidden_states)
+        all_attentions = ()
+
+        if self.output_hidden_states:
+            all_hidden_states = (hidden_states,)
+
+        for i in range(self.config.num_hidden_layers):
+            # Number of layers in a hidden group
+            layers_per_group = int(
+                self.config.num_hidden_layers / self.config.num_hidden_groups)
+
+            # Index of the hidden group
+            group_idx = int(
+                i / (self.config.num_hidden_layers / self.config.num_hidden_groups))
+
+            layer_group_output = self.albert_layer_groups[group_idx](
+                [hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]], training=training)
+            hidden_states = layer_group_output[0]
+
+            if self.output_attentions:
+                all_attentions = all_attentions + layer_group_output[-1]
+
+            if self.output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (all_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (all_attentions,)
+
+        # last-layer hidden state, (all hidden states), (all attentions)
+        return outputs
+
+
+class TFAlbertPreTrainedModel(TFPreTrainedModel):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+    config_class = AlbertConfig
+    pretrained_model_archive_map = TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "albert"
+
+
+class TFAlbertMLMHead(tf.keras.layers.Layer):
+    def __init__(self, config, input_embeddings, **kwargs):
+        super(TFAlbertMLMHead, self).__init__(**kwargs)
+        self.vocab_size = config.vocab_size
+
+        self.dense = tf.keras.layers.Dense(config.embedding_size,
+                                           kernel_initializer=get_initializer(
+                                               config.initializer_range),
+                                           name='dense')
+        if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)):
+            self.activation = ACT2FN[config.hidden_act]
+        else:
+            self.activation = config.hidden_act
+
+        self.LayerNorm = tf.keras.layers.LayerNormalization(
+            epsilon=config.layer_norm_eps, name='LayerNorm')
+
+        # The output weights are the same as the input embeddings, but there is
+        # an output-only bias for each token.
+        self.input_embeddings = input_embeddings
+
+    def build(self, input_shape):
+        self.bias = self.add_weight(shape=(self.vocab_size,),
+                                    initializer='zeros',
+                                    trainable=True,
+                                    name='bias')
+        super(TFAlbertMLMHead, self).build(input_shape)
+
+    def call(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
+        hidden_states = self.input_embeddings(hidden_states, mode="linear")
+        hidden_states = hidden_states + self.bias
+        return hidden_states
+
+
+ALBERT_START_DOCSTRING = r"""    The ALBERT model was proposed in
+    `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_
+    by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer
+    pre-trained using a combination of masked language modeling objective and next sentence prediction
+    on a large corpus comprising the Toronto Book Corpus and Wikipedia.
+
+    This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and
+    refer to the TF 2.0 documentation for all matter related to general usage and behavior.
+
+    .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`:
+        https://arxiv.org/abs/1810.04805
+
+    .. _`tf.keras.Model`:
+        https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model
+
+    Note on the model inputs:
+        TF 2.0 models accepts two formats as inputs:
+
+            - having all inputs as keyword arguments (like PyTorch models), or
+            - having all inputs as a list, tuple or dict in the first positional arguments.
+
+        This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`.
+
+        If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument :
+
+        - a single Tensor with input_ids only and nothing else: `model(inputs_ids)
+        - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
+            `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])`
+        - a dictionary with one or several input Tensors associaed to the input names given in the docstring:
+            `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})`
+
+    Parameters:
+        config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. 
+            Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
+"""
+
+ALBERT_INPUTS_DOCSTRING = r"""
+    Inputs:
+        **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of input sequence tokens in the vocabulary.
+            To match pre-training, ALBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows:
+
+            (a) For sequence pairs:
+
+                ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]``
+                
+                ``token_type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1``
+
+            (b) For single sequences:
+
+                ``tokens:         [CLS] the dog is hairy . [SEP]``
+                
+                ``token_type_ids:   0   0   0   0  0     0   0``
+
+            Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on
+            the right rather than the left.
+
+            Indices can be obtained using :class:`transformers.AlbertTokenizer`.
+            See :func:`transformers.PreTrainedTokenizer.encode` and
+            :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
+        **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
+        **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Segment token indices to indicate first and second portions of the inputs.
+            Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1``
+            corresponds to a `sentence B` token
+            (see `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details).
+        **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``:
+            Indices of positions of each input sequence tokens in the position embeddings.
+            Selected in the range ``[0, config.max_position_embeddings - 1]``.
+        **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:
+            Mask to nullify selected heads of the self-attention modules.
+            Mask values selected in ``[0, 1]``:
+            ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
+"""
+
+@add_start_docstrings("The bare Albert Model transformer outputing raw hidden-states without any specific head on top.",
+                      ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
+class TFAlbertModel(TFAlbertPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)``
+            Sequence of hidden-states at the output of the last layer of the model.
+        **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)``
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during Albert pretraining. This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import AlbertTokenizer, TFAlbertModel
+
+        tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased')
+        model = TFAlbertModel.from_pretrained('bert-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
+    """
+
+    def __init__(self, config, **kwargs):
+        super(TFAlbertModel, self).__init__(config, **kwargs)
+        self.num_hidden_layers = config.num_hidden_layers
+
+        self.embeddings = TFAlbertEmbeddings(config, name="embeddings")
+        self.encoder = TFAlbertTransformer(config, name="encoder")
+        self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer(
+            config.initializer_range), activation='tanh', name='pooler')
+
+    def _resize_token_embeddings(self, new_num_tokens):
+        raise NotImplementedError
+
+    def _prune_heads(self, heads_to_prune):
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            See base class PreTrainedModel
+        """
+        raise NotImplementedError
+
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+        if isinstance(inputs, (tuple, list)):
+            input_ids = inputs[0]
+            attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
+            token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
+            position_ids = inputs[3] if len(inputs) > 3 else position_ids
+            head_mask = inputs[4] if len(inputs) > 4 else head_mask
+            assert len(inputs) <= 5, "Too many inputs."
+        elif isinstance(inputs, dict):
+            input_ids = inputs.get('input_ids')
+            attention_mask = inputs.get('attention_mask', attention_mask)
+            token_type_ids = inputs.get('token_type_ids', token_type_ids)
+            position_ids = inputs.get('position_ids', position_ids)
+            head_mask = inputs.get('head_mask', head_mask)
+            assert len(inputs) <= 5, "Too many inputs."
+        else:
+            input_ids = inputs
+
+        if attention_mask is None:
+            attention_mask = tf.fill(tf.shape(input_ids), 1)
+        if token_type_ids is None:
+            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+
+        # We create a 3D attention mask from a 2D tensor mask.
+        # Sizes are [batch_size, 1, 1, to_seq_length]
+        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+        # this attention mask is more simple than the triangular masking of causal attention
+        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+        extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :]
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+
+        extended_attention_mask = tf.cast(extended_attention_mask, tf.float32)
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        if not head_mask is None:
+            raise NotImplementedError
+        else:
+            head_mask = [None] * self.num_hidden_layers
+            # head_mask = tf.constant([0] * self.num_hidden_layers)
+
+        embedding_output = self.embeddings(
+            [input_ids, position_ids, token_type_ids], training=training)
+        encoder_outputs = self.encoder(
+            [embedding_output, extended_attention_mask, head_mask], training=training)
+
+        sequence_output = encoder_outputs[0]
+        pooled_output = self.pooler(sequence_output)
+
+        # add hidden_states and attentions if they are here
+        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]
+        # sequence_output, pooled_output, (hidden_states), (attentions)
+        return outputs
+
+
+@add_start_docstrings("""Albert Model with a `language modeling` head on top. """,
+                      ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
+class TFAlbertForMaskedLM(TFAlbertPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import AlbertTokenizer, TFAlbertForMaskedLM
+
+        tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased')
+        model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        prediction_scores = outputs[0]
+
+    """
+
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFAlbertForMaskedLM, self).__init__(config, *inputs, **kwargs)
+
+        self.albert = TFAlbertModel(config, name='albert')
+        self.predictions = TFAlbertMLMHead(
+            config, self.albert.embeddings, name='predictions')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.albert(inputs, **kwargs)
+
+        sequence_output = outputs[0]
+        prediction_scores = self.predictions(
+            sequence_output, training=kwargs.get('training', False))
+
+        # Add hidden states and attention if they are here
+        outputs = (prediction_scores,) + outputs[2:]
+
+        return outputs  # prediction_scores, (hidden_states), (attentions)

From 7bddbf5961f35d03db5151a4747085f7a57d7ff7 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Thu, 7 Nov 2019 23:50:05 +0000
Subject: [PATCH 231/269] TFAlbertForSequenceClassification

---
 transformers/modeling_tf_albert.py | 66 ++++++++++++++++++++++++++----
 1 file changed, 58 insertions(+), 8 deletions(-)

diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py
index 8861b7add8..410d83ff78 100644
--- a/transformers/modeling_tf_albert.py
+++ b/transformers/modeling_tf_albert.py
@@ -479,16 +479,15 @@ class TFAlbertMLMHead(tf.keras.layers.Layer):
 
 
 ALBERT_START_DOCSTRING = r"""    The ALBERT model was proposed in
-    `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_
-    by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer
-    pre-trained using a combination of masked language modeling objective and next sentence prediction
-    on a large corpus comprising the Toronto Book Corpus and Wikipedia.
+    `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_
+    by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents
+    two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT.
 
     This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and
     refer to the TF 2.0 documentation for all matter related to general usage and behavior.
 
-    .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`:
-        https://arxiv.org/abs/1810.04805
+    .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`:
+        https://arxiv.org/abs/1909.11942
 
     .. _`tf.keras.Model`:
         https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model
@@ -695,8 +694,8 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel):
         import tensorflow as tf
         from transformers import AlbertTokenizer, TFAlbertForMaskedLM
 
-        tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased')
-        model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased')
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = TFAlbertForMaskedLM.from_pretrained('albert-base-v2')
         input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
         outputs = model(input_ids)
         prediction_scores = outputs[0]
@@ -721,3 +720,54 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel):
         outputs = (prediction_scores,) + outputs[2:]
 
         return outputs  # prediction_scores, (hidden_states), (attentions)
+
+
+@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of
+    the pooled output) e.g. for GLUE tasks. """,
+    ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING)
+class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel):
+    r"""
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        import tensorflow as tf
+        from transformers import AlbertTokenizer, TFAlbertForSequenceClassification
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = TFAlbertForSequenceClassification.from_pretrained('albert-base-v2')
+        input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :]  # Batch size 1
+        outputs = model(input_ids)
+        logits = outputs[0]
+
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(TFAlbertForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        self.num_labels = config.num_labels
+
+        self.albert = TFAlbertModel(config, name='albert')
+        self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
+        self.classifier = tf.keras.layers.Dense(config.num_labels,
+                                                kernel_initializer=get_initializer(config.initializer_range),
+                                                name='classifier')
+
+    def call(self, inputs, **kwargs):
+        outputs = self.albert(inputs, **kwargs)
+
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False))
+        logits = self.classifier(pooled_output)
+
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        return outputs  # logits, (hidden_states), (attentions)
\ No newline at end of file

From b18509c2085dce16e655dd86b7ea0129335da4e1 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Fri, 8 Nov 2019 00:12:21 +0000
Subject: [PATCH 232/269] Tests for ALBERT in TF2 + fixes

---
 transformers/__init__.py                      |   1 +
 transformers/modeling_tf_albert.py            |  18 +-
 transformers/tests/modeling_tf_albert_test.py | 229 ++++++++++++++++++
 3 files changed, 237 insertions(+), 11 deletions(-)
 create mode 100644 transformers/tests/modeling_tf_albert_test.py

diff --git a/transformers/__init__.py b/transformers/__init__.py
index a409ef772e..baf430c17b 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -169,6 +169,7 @@ if is_tf_available():
                                     TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP)
 
     from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM,
+                                     TFAlbertForSequenceClassification,
                                     TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
 
 # TF 2.0 <=> PyTorch conversion utilities
diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py
index 410d83ff78..a3f183b192 100644
--- a/transformers/modeling_tf_albert.py
+++ b/transformers/modeling_tf_albert.py
@@ -126,16 +126,8 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer):
         """
         batch_size = tf.shape(inputs)[0]
         length = tf.shape(inputs)[1]
-
-        print(inputs.shape)
-
         x = tf.reshape(inputs, [-1, self.config.embedding_size])
-
-        print(x.shape, self.word_embeddings)
-
         logits = tf.matmul(x, self.word_embeddings, transpose_b=True)
-
-        print([batch_size, length, self.config.vocab_size])
         return tf.reshape(logits, [batch_size, length, self.config.vocab_size])
 
 
@@ -460,20 +452,24 @@ class TFAlbertMLMHead(tf.keras.layers.Layer):
 
         # The output weights are the same as the input embeddings, but there is
         # an output-only bias for each token.
-        self.input_embeddings = input_embeddings
+        self.decoder = input_embeddings
 
     def build(self, input_shape):
         self.bias = self.add_weight(shape=(self.vocab_size,),
                                     initializer='zeros',
                                     trainable=True,
                                     name='bias')
+        self.decoder_bias = self.add_weight(shape=(self.vocab_size,),
+                                    initializer='zeros',
+                                    trainable=True,
+                                    name='decoder/bias')
         super(TFAlbertMLMHead, self).build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.dense(hidden_states)
         hidden_states = self.activation(hidden_states)
         hidden_states = self.LayerNorm(hidden_states)
-        hidden_states = self.input_embeddings(hidden_states, mode="linear")
+        hidden_states = self.decoder(hidden_states, mode="linear") + self.decoder_bias
         hidden_states = hidden_states + self.bias
         return hidden_states
 
@@ -666,7 +662,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
             [embedding_output, extended_attention_mask, head_mask], training=training)
 
         sequence_output = encoder_outputs[0]
-        pooled_output = self.pooler(sequence_output)
+        pooled_output = self.pooler(sequence_output[:, 0])
 
         # add hidden_states and attentions if they are here
         outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]
diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py
new file mode 100644
index 0000000000..85fc62f34f
--- /dev/null
+++ b/transformers/tests/modeling_tf_albert_test.py
@@ -0,0 +1,229 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+import shutil
+import pytest
+import sys
+
+from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor)
+from .configuration_common_test import ConfigTester
+
+from transformers import AlbertConfig, is_tf_available
+
+if is_tf_available():
+    import tensorflow as tf
+    from transformers.modeling_tf_albert import (TFAlbertModel, TFAlbertForMaskedLM,
+                                                 TFAlbertForSequenceClassification,
+                                                 TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP)
+else:
+    pytestmark = pytest.mark.skip("Require TensorFlow")
+
+
+class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester):
+
+    all_model_classes = (
+        TFAlbertModel,
+        TFAlbertForMaskedLM,
+        TFAlbertForSequenceClassification
+    ) if is_tf_available() else ()
+
+    class TFAlbertModelTester(object):
+
+        def __init__(self,
+                     parent,
+                     batch_size=13,
+                     seq_length=7,
+                     is_training=True,
+                     use_input_mask=True,
+                     use_token_type_ids=True,
+                     use_labels=True,
+                     vocab_size=99,
+                     hidden_size=32,
+                     num_hidden_layers=5,
+                     num_attention_heads=4,
+                     intermediate_size=37,
+                     hidden_act="gelu",
+                     hidden_dropout_prob=0.1,
+                     attention_probs_dropout_prob=0.1,
+                     max_position_embeddings=512,
+                     type_vocab_size=16,
+                     type_sequence_label_size=2,
+                     initializer_range=0.02,
+                     num_labels=3,
+                     num_choices=4,
+                     scope=None,
+                     ):
+            self.parent = parent
+            self.batch_size = batch_size
+            self.seq_length = seq_length
+            self.is_training = is_training
+            self.use_input_mask = use_input_mask
+            self.use_token_type_ids = use_token_type_ids
+            self.use_labels = use_labels
+            self.vocab_size = vocab_size
+            self.hidden_size = hidden_size
+            self.num_hidden_layers = num_hidden_layers
+            self.num_attention_heads = num_attention_heads
+            self.intermediate_size = intermediate_size
+            self.hidden_act = hidden_act
+            self.hidden_dropout_prob = hidden_dropout_prob
+            self.attention_probs_dropout_prob = attention_probs_dropout_prob
+            self.max_position_embeddings = max_position_embeddings
+            self.type_vocab_size = type_vocab_size
+            self.type_sequence_label_size = type_sequence_label_size
+            self.initializer_range = initializer_range
+            self.num_labels = num_labels
+            self.num_choices = num_choices
+            self.scope = scope
+
+        def prepare_config_and_inputs(self):
+            input_ids = ids_tensor(
+                [self.batch_size, self.seq_length], self.vocab_size)
+
+            input_mask = None
+            if self.use_input_mask:
+                input_mask = ids_tensor(
+                    [self.batch_size, self.seq_length], vocab_size=2)
+
+            token_type_ids = None
+            if self.use_token_type_ids:
+                token_type_ids = ids_tensor(
+                    [self.batch_size, self.seq_length], self.type_vocab_size)
+
+            sequence_labels = None
+            token_labels = None
+            choice_labels = None
+            if self.use_labels:
+                sequence_labels = ids_tensor(
+                    [self.batch_size], self.type_sequence_label_size)
+                token_labels = ids_tensor(
+                    [self.batch_size, self.seq_length], self.num_labels)
+                choice_labels = ids_tensor([self.batch_size], self.num_choices)
+
+            config = AlbertConfig(
+                vocab_size_or_config_json_file=self.vocab_size,
+                hidden_size=self.hidden_size,
+                num_hidden_layers=self.num_hidden_layers,
+                num_attention_heads=self.num_attention_heads,
+                intermediate_size=self.intermediate_size,
+                hidden_act=self.hidden_act,
+                hidden_dropout_prob=self.hidden_dropout_prob,
+                attention_probs_dropout_prob=self.attention_probs_dropout_prob,
+                max_position_embeddings=self.max_position_embeddings,
+                type_vocab_size=self.type_vocab_size,
+                initializer_range=self.initializer_range)
+
+            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+
+        def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = TFAlbertModel(config=config)
+            # inputs = {'input_ids': input_ids,
+            #           'attention_mask': input_mask,
+            #           'token_type_ids': token_type_ids}
+            # sequence_output, pooled_output = model(**inputs)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            sequence_output, pooled_output = model(inputs)
+
+            inputs = [input_ids, input_mask]
+            sequence_output, pooled_output = model(inputs)
+
+            sequence_output, pooled_output = model(input_ids)
+
+            result = {
+                "sequence_output": sequence_output.numpy(),
+                "pooled_output": pooled_output.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["sequence_output"].shape),
+                [self.batch_size, self.seq_length, self.hidden_size])
+            self.parent.assertListEqual(list(result["pooled_output"].shape), [
+                                        self.batch_size, self.hidden_size])
+
+        def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            model = TFAlbertForMaskedLM(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            prediction_scores, = model(inputs)
+            result = {
+                "prediction_scores": prediction_scores.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["prediction_scores"].shape),
+                [self.batch_size, self.seq_length, self.vocab_size])
+
+        def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = TFAlbertForSequenceClassification(config=config)
+            inputs = {'input_ids': input_ids,
+                      'attention_mask': input_mask,
+                      'token_type_ids': token_type_ids}
+            logits, = model(inputs)
+            result = {
+                "logits": logits.numpy(),
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].shape),
+                [self.batch_size, self.num_labels])
+
+        def prepare_config_and_inputs_for_common(self):
+            config_and_inputs = self.prepare_config_and_inputs()
+            (config, input_ids, token_type_ids, input_mask,
+             sequence_labels, token_labels, choice_labels) = config_and_inputs
+            inputs_dict = {'input_ids': input_ids,
+                           'token_type_ids': token_type_ids, 'attention_mask': input_mask}
+            return config, inputs_dict
+
+    def setUp(self):
+        self.model_tester = TFAlbertModelTest.TFAlbertModelTester(self)
+        self.config_tester = ConfigTester(
+            self, config_class=AlbertConfig, hidden_size=37)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_albert_model(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_model(*config_and_inputs)
+
+    def test_for_masked_lm(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_for_masked_lm(
+            *config_and_inputs)
+
+    def test_for_sequence_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_albert_for_sequence_classification(
+            *config_and_inputs)
+
+    @pytest.mark.slow
+    def test_model_from_pretrained(self):
+        cache_dir = "/tmp/transformers_test/"
+        # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
+        for model_name in ['albert-base-uncased']:
+            model = TFAlbertModel.from_pretrained(
+                model_name, cache_dir=cache_dir)
+            shutil.rmtree(cache_dir)
+            self.assertIsNotNone(model)
+
+
+if __name__ == "__main__":
+    unittest.main()

From c9cb7f8a0fbe784665b00bfdca6bfc54ad10d5f5 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Mon, 11 Nov 2019 15:12:54 -0500
Subject: [PATCH 233/269] Torch 1.1.0 compatibility + FP16 O1 + TF checkpoints

Co-authored-by: wassname
---
 transformers/modeling_albert.py    | 4 ++--
 transformers/modeling_tf_albert.py | 9 ++++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 640af537d0..ff20ca78dc 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -203,8 +203,8 @@ class AlbertAttention(BertSelfAttention):
         
 
         # Should find a better way to do this
-        w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size)
-        b = self.dense.bias
+        w = self.dense.weight.t().view(self.num_attention_heads, self.attention_head_size, self.hidden_size).to(context_layer.dtype)
+        b = self.dense.bias.to(context_layer.dtype)
 
         projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
         projected_context_layer_dropout = self.dropout(projected_context_layer)
diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py
index a3f183b192..ee8712eb28 100644
--- a/transformers/modeling_tf_albert.py
+++ b/transformers/modeling_tf_albert.py
@@ -36,7 +36,14 @@ import logging
 logger = logging.getLogger(__name__)
 
 TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    # TODO FILL THAT UP
+    'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-tf_model.h5",
+    'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-tf_model.h5",
+    'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-tf_model.h5",
+    'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-tf_model.h5",
+    'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-tf_model.h5",
+    'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-tf_model.h5",
+    'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-tf_model.h5",
+    'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-tf_model.h5",
 }
 
 

From f873b55e433e1c9ef364e597ad1d4662db4bc038 Mon Sep 17 00:00:00 2001
From: LysandreJik <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 10:28:41 -0500
Subject: [PATCH 234/269] Warning for ALBERT-v2 models

---
 transformers/modeling_utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py
index d51eefab58..5d51caa951 100644
--- a/transformers/modeling_utils.py
+++ b/transformers/modeling_utils.py
@@ -315,6 +315,10 @@ class PreTrainedModel(nn.Module):
             model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
+        if "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path:
+            logger.warning("There is currently an upstream reproducibility issue with ALBERT v2 models. Please see " +
+                           "https://github.com/google-research/google-research/issues/119 for more information.")
+
         config = kwargs.pop('config', None)
         state_dict = kwargs.pop('state_dict', None)
         cache_dir = kwargs.pop('cache_dir', None)

From c536c2a4809a51356fd50ff62a80955fea79f790 Mon Sep 17 00:00:00 2001
From: LysandreJik <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 11:22:52 -0500
Subject: [PATCH 235/269] ALBERT Input Embeds

---
 transformers/modeling_albert.py    | 77 ++++++++++++++++++++++--------
 transformers/modeling_tf_albert.py | 45 ++++++++++++-----
 2 files changed, 90 insertions(+), 32 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index ff20ca78dc..7882356d24 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -433,6 +433,12 @@ class AlbertModel(AlbertPreTrainedModel):
 
         self.init_weights()
 
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
     def _resize_token_embeddings(self, new_num_tokens):
         old_embeddings = self.embeddings.word_embeddings
         new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
@@ -457,12 +463,24 @@ class AlbertModel(AlbertPreTrainedModel):
             inner_group_idx = int(layer - group_idx * self.config.inner_group_num)
             self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads)
 
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                inputs_embeds=None):
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
         if attention_mask is None:
-            attention_mask = torch.ones_like(input_ids)
+            attention_mask = torch.ones(input_shape, device=device)
         if token_type_ids is None:
-            token_type_ids = torch.zeros_like(input_ids)
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
 
         extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
         extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility
@@ -477,7 +495,8 @@ class AlbertModel(AlbertPreTrainedModel):
         else:
             head_mask = [None] * self.config.num_hidden_layers
 
-        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids)
+        embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids,
+                                           inputs_embeds=inputs_embeds)
         encoder_outputs = self.encoder(embedding_output,
                                        extended_attention_mask,
                                        head_mask=head_mask)
@@ -549,9 +568,19 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
         self._tie_or_clone_weights(self.predictions.decoder,
                                    self.albert.embeddings.word_embeddings)
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None):
-        outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask)
+    def get_output_embeddings(self):
+        return self.predictions.decoder
+
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                masked_lm_labels=None, inputs_embeds=None):
+        outputs = self.albert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds
+        )
         sequence_outputs = outputs[0]
 
         prediction_scores = self.predictions(sequence_outputs)
@@ -609,14 +638,17 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
-                position_ids=None, head_mask=None, labels=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
 
-        outputs = self.albert(input_ids,
-                            attention_mask=attention_mask,
-                            token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
-                            head_mask=head_mask)
+        outputs = self.albert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds
+        )
 
         pooled_output = outputs[1]
 
@@ -692,14 +724,17 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                start_positions=None, end_positions=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
+                inputs_embeds=None, start_positions=None, end_positions=None):
 
-        outputs = self.albert(input_ids,
-                            attention_mask=attention_mask,
-                            token_type_ids=token_type_ids,
-                            position_ids=position_ids, 
-                            head_mask=head_mask)
+        outputs = self.albert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds
+        )
 
         sequence_output = outputs[0]
 
diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py
index ee8712eb28..ee122205b9 100644
--- a/transformers/modeling_tf_albert.py
+++ b/transformers/modeling_tf_albert.py
@@ -107,19 +107,25 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer):
 
     def _embedding(self, inputs, training=False):
         """Applies embedding based on inputs tensor."""
-        input_ids, position_ids, token_type_ids = inputs
+        input_ids, position_ids, token_type_ids, inputs_embeds = inputs
 
-        seq_length = tf.shape(input_ids)[1]
+        if input_ids is not None:
+            input_shape = tf.shape(input_ids)
+        else:
+            input_shape = tf.shape(inputs_embeds)[:-1]
+
+        seq_length = input_shape[1]
         if position_ids is None:
             position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :]
         if token_type_ids is None:
-            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+            token_type_ids = tf.fill(input_shape, 0)
 
-        words_embeddings = tf.gather(self.word_embeddings, input_ids)
+        if inputs_embeds is None:
+            inputs_embeds = tf.gather(self.word_embeddings, input_ids)
         position_embeddings = self.position_embeddings(position_ids)
         token_type_embeddings = self.token_type_embeddings(token_type_ids)
 
-        embeddings = words_embeddings + position_embeddings + token_type_embeddings
+        embeddings = inputs_embeds + position_embeddings + token_type_embeddings
         embeddings = self.LayerNorm(embeddings)
         embeddings = self.dropout(embeddings, training=training)
         return embeddings
@@ -603,6 +609,9 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
         self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer(
             config.initializer_range), activation='tanh', name='pooler')
 
+    def get_input_embeddings(self):
+        return self.embeddings
+
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError
 
@@ -613,28 +622,39 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
         """
         raise NotImplementedError
 
-    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False):
+    def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False):
         if isinstance(inputs, (tuple, list)):
             input_ids = inputs[0]
             attention_mask = inputs[1] if len(inputs) > 1 else attention_mask
             token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids
             position_ids = inputs[3] if len(inputs) > 3 else position_ids
             head_mask = inputs[4] if len(inputs) > 4 else head_mask
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds
+            assert len(inputs) <= 6, "Too many inputs."
         elif isinstance(inputs, dict):
             input_ids = inputs.get('input_ids')
             attention_mask = inputs.get('attention_mask', attention_mask)
             token_type_ids = inputs.get('token_type_ids', token_type_ids)
             position_ids = inputs.get('position_ids', position_ids)
             head_mask = inputs.get('head_mask', head_mask)
-            assert len(inputs) <= 5, "Too many inputs."
+            inputs_embeds = inputs.get('inputs_embeds', inputs_embeds)
+            assert len(inputs) <= 6, "Too many inputs."
         else:
             input_ids = inputs
 
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.shape
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.shape[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
         if attention_mask is None:
-            attention_mask = tf.fill(tf.shape(input_ids), 1)
+            attention_mask = tf.fill(input_shape, 1)
         if token_type_ids is None:
-            token_type_ids = tf.fill(tf.shape(input_ids), 0)
+            token_type_ids = tf.fill(input_shape, 0)
 
         # We create a 3D attention mask from a 2D tensor mask.
         # Sizes are [batch_size, 1, 1, to_seq_length]
@@ -664,7 +684,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
             # head_mask = tf.constant([0] * self.num_hidden_layers)
 
         embedding_output = self.embeddings(
-            [input_ids, position_ids, token_type_ids], training=training)
+            [input_ids, position_ids, token_type_ids, inputs_embeds], training=training)
         encoder_outputs = self.encoder(
             [embedding_output, extended_attention_mask, head_mask], training=training)
 
@@ -712,6 +732,9 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel):
         self.predictions = TFAlbertMLMHead(
             config, self.albert.embeddings, name='predictions')
 
+    def get_output_embeddings(self):
+        return self.albert.embeddings
+
     def call(self, inputs, **kwargs):
         outputs = self.albert(inputs, **kwargs)
 

From bdfe21ab242c2175972f53ca6b4af29ada02a2bb Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 26 Nov 2019 12:54:36 -0500
Subject: [PATCH 236/269] Change param order for consistency

---
 transformers/modeling_albert.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py
index 7882356d24..5b7b2d3900 100644
--- a/transformers/modeling_albert.py
+++ b/transformers/modeling_albert.py
@@ -571,8 +571,8 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
     def get_output_embeddings(self):
         return self.predictions.decoder
 
-    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
-                masked_lm_labels=None, inputs_embeds=None):
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
+                masked_lm_labels=None):
         outputs = self.albert(
             input_ids=input_ids,
             attention_mask=attention_mask,

From f2f329408db66285fd59e6628ca394381bb7f94e Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 12:59:28 -0500
Subject: [PATCH 237/269] Fix input embeddings

---
 transformers/tests/modeling_albert_test.py    | 2 ++
 transformers/tests/modeling_tf_albert_test.py | 2 ++
 transformers/tests/modeling_tf_common_test.py | 7 ++++---
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py
index da87709df1..976feff9db 100644
--- a/transformers/tests/modeling_albert_test.py
+++ b/transformers/tests/modeling_albert_test.py
@@ -49,6 +49,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
                      use_token_type_ids=True,
                      use_labels=True,
                      vocab_size=99,
+                     embedding_size=16,
                      hidden_size=36,
                      num_hidden_layers=6,
                      num_hidden_groups=6,
@@ -73,6 +74,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester):
             self.use_token_type_ids = use_token_type_ids
             self.use_labels = use_labels
             self.vocab_size = vocab_size
+            self.embedding_size = embedding_size
             self.hidden_size = hidden_size
             self.num_hidden_layers = num_hidden_layers
             self.num_attention_heads = num_attention_heads
diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py
index 85fc62f34f..fbd519b8f6 100644
--- a/transformers/tests/modeling_tf_albert_test.py
+++ b/transformers/tests/modeling_tf_albert_test.py
@@ -54,6 +54,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester):
                      use_token_type_ids=True,
                      use_labels=True,
                      vocab_size=99,
+                     embedding_size=16,
                      hidden_size=32,
                      num_hidden_layers=5,
                      num_attention_heads=4,
@@ -77,6 +78,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester):
             self.use_token_type_ids = use_token_type_ids
             self.use_labels = use_labels
             self.vocab_size = vocab_size
+            self.embedding_size = embedding_size
             self.hidden_size = hidden_size
             self.num_hidden_layers = num_hidden_layers
             self.num_attention_heads = num_attention_heads
diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py
index 2bb7cc9c5f..31a30766cf 100644
--- a/transformers/tests/modeling_tf_common_test.py
+++ b/transformers/tests/modeling_tf_common_test.py
@@ -426,9 +426,10 @@ class TFCommonTestCases:
                     try:
                         x = wte([input_ids], mode="embedding")
                     except:
-                        x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
-                # ^^ In our TF models, the input_embeddings can take slightly different forms,
-                # so we try two of them and fall back to just synthetically creating a dummy tensor of ones.
+                        if hasattr(self.model_tester, "embedding_size"):
+                            x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32)
+                        else:
+                            x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
                 inputs_dict["inputs_embeds"] = x
                 outputs = model(inputs_dict)
 

From ae98d4599179b299563b679dd33f8a86da12980d Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 14:12:44 -0500
Subject: [PATCH 238/269] Release: v2.2.0

---
 .circleci/deploy.sh         |  1 +
 README.md                   |  2 +-
 deploy_multi_version_doc.sh | 22 ++++++++++++++++++++++
 docs/source/conf.py         |  2 +-
 setup.py                    |  2 +-
 transformers/__init__.py    |  2 +-
 6 files changed, 27 insertions(+), 4 deletions(-)
 create mode 100644 deploy_multi_version_doc.sh

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 98151963d8..c4b802c9e9 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -23,3 +23,4 @@ deploy_doc "fe02e45" v1.1.0
 deploy_doc "89fd345" v1.2.0
 deploy_doc "fc9faa8" v2.0.0
 deploy_doc "3ddce1d" v2.1.1
+deploy_doc "f2f3294" v2.2.0
diff --git a/README.md b/README.md
index 52e74a6f80..bb6a8eaa1a 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime
 | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation |
 | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers |
 | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers |
-| [Documentation](https://huggingface.co/transformers/) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more |
+| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more |
 
 ## Installation
 
diff --git a/deploy_multi_version_doc.sh b/deploy_multi_version_doc.sh
new file mode 100644
index 0000000000..bd567213eb
--- /dev/null
+++ b/deploy_multi_version_doc.sh
@@ -0,0 +1,22 @@
+cd docs
+
+function deploy_doc(){
+	echo "Creating doc at commit $1 and pushing to folder $2"
+	git checkout $1
+	if [ ! -z "$2" ] 
+	then
+		echo "Pushing version" $2
+		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+	else
+		echo "Pushing master"
+		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir
+	fi
+}
+
+deploy_doc "master" 
+deploy_doc "b33a385" v1.0.0
+deploy_doc "fe02e45" v1.1.0
+deploy_doc "89fd345" v1.2.0
+deploy_doc "fc9faa8" v2.0.0
+deploy_doc "3ddce1d" v2.1.1
+deploy_doc "f2f3294" v2.2.0
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 00c020ab39..f762a89cd2 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'2.1.1'
+release = u'2.2.0'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/setup.py b/setup.py
index f49aee68d4..d8dcf7b898 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="transformers",
-    version="2.1.1",
+    version="2.2.0",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
diff --git a/transformers/__init__.py b/transformers/__init__.py
index baf430c17b..dff2479d4b 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.1.1"
+__version__ = "2.2.0"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.

From b6321452738925983ba1306fb5396982d64f408a Mon Sep 17 00:00:00 2001
From: Lysandre Debut <lysandre@huggingface.co>
Date: Tue, 26 Nov 2019 14:27:15 -0500
Subject: [PATCH 239/269] Update master documentation link in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index bb6a8eaa1a..67fb80aecf 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime
 | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation |
 | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers |
 | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers |
-| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more |
+| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more |
 
 ## Installation
 

From cf62bdc962c53d9fb7a5820217f1bf844bb6da3b Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 26 Nov 2019 14:37:32 -0500
Subject: [PATCH 240/269] Improve test protocol for inputs_embeds in TF

cc @lysandrejik
---
 transformers/tests/modeling_tf_common_test.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py
index 31a30766cf..232991915c 100644
--- a/transformers/tests/modeling_tf_common_test.py
+++ b/transformers/tests/modeling_tf_common_test.py
@@ -426,10 +426,15 @@ class TFCommonTestCases:
                     try:
                         x = wte([input_ids], mode="embedding")
                     except:
-                        if hasattr(self.model_tester, "embedding_size"):
-                            x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32)
-                        else:
-                            x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
+                        x = wte([input_ids, None, None, None], mode="embedding")
+                # ^^ In our TF models, the input_embeddings can take slightly different forms,
+                # so we try a few of them.
+                # We used to fall back to just synthetically creating a dummy tensor of ones:
+                #
+                # if hasattr(self.model_tester, "embedding_size"):
+                #     x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32)
+                # else:
+                #     x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
                 inputs_dict["inputs_embeds"] = x
                 outputs = model(inputs_dict)
 

From 8742baa53136b906e392fdae57f1c191d1b2370e Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Tue, 26 Nov 2019 14:39:47 -0500
Subject: [PATCH 241/269] Improve test protocol for inputs_embeds in TF

---
 transformers/tests/modeling_tf_common_test.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py
index 232991915c..ea8cd1aecd 100644
--- a/transformers/tests/modeling_tf_common_test.py
+++ b/transformers/tests/modeling_tf_common_test.py
@@ -426,15 +426,17 @@ class TFCommonTestCases:
                     try:
                         x = wte([input_ids], mode="embedding")
                     except:
-                        x = wte([input_ids, None, None, None], mode="embedding")
+                        try:
+                            x = wte([input_ids, None, None, None], mode="embedding")
+                        except:
+                            if hasattr(self.model_tester, "embedding_size"):
+                                x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32)
+                            else:
+                                x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
                 # ^^ In our TF models, the input_embeddings can take slightly different forms,
                 # so we try a few of them.
                 # We used to fall back to just synthetically creating a dummy tensor of ones:
                 #
-                # if hasattr(self.model_tester, "embedding_size"):
-                #     x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32)
-                # else:
-                #     x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
                 inputs_dict["inputs_embeds"] = x
                 outputs = model(inputs_dict)
 

From 668aac45d206bc0c8ca95e7a8b565d1ff701da7f Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 14:52:42 -0500
Subject: [PATCH 242/269] Pretrained models

---
 docs/source/pretrained_models.rst | 33 +++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index b0a578fd80..d017a3be6c 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -159,5 +159,38 @@ Here is the full list of the currently provided pretrained models together with
 |                   |                                                            | | CamemBERT using the BERT-base architecture                                                                                          |
 |                   |                                                            | (see `details <https://github.com/pytorch/fairseq/tree/master/examples/camembert>`__)                                                 |
 +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+| ALBERT            | ``albert-base-v1``                                         | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters                                                            |
+|                   |                                                            | | ALBERT base model                                                                                                                   |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-large-v1``                                        | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters                                                           |
+|                   |                                                            | | ALBERT large model                                                                                                                  |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-xlarge-v1``                                       | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters                                                           |
+|                   |                                                            | | ALBERT xlarge model                                                                                                                 |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-xxlarge-v1``                                      | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters                                                           |
+|                   |                                                            | | ALBERT xxlarge model                                                                                                                |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-base-v2``                                         | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters                                                            |
+|                   |                                                            | | ALBERT base model with no dropout, additional training data and longer training                                                     |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-large-v2``                                        | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters                                                           |
+|                   |                                                            | | ALBERT large model with no dropout, additional training data and longer training                                                    |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-xlarge-v2``                                       | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters                                                           |
+|                   |                                                            | | ALBERT xlarge model with no dropout, additional training data and longer training                                                   |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   | ``albert-xxlarge-v2``                                      | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters                                                           |
+|                   |                                                            | | ALBERT xxlarge model with no dropout, additional training data and longer training                                                  |
+|                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+
 
 .. <https://huggingface.co/transformers/examples.html>`__

From 7c6000e412288ddce7d0dbc4f4d9a12f49d28690 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 14:55:29 -0500
Subject: [PATCH 243/269] Updated v2.2.0 doc

---
 .circleci/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index c4b802c9e9..73960cbe99 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0
 deploy_doc "89fd345" v1.2.0
 deploy_doc "fc9faa8" v2.0.0
 deploy_doc "3ddce1d" v2.1.1
-deploy_doc "f2f3294" v2.2.0
+deploy_doc "668aac4" v2.2.0

From ee4647bd5c1343304f7fdf20f06db1fc2524f892 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 15:10:51 -0500
Subject: [PATCH 244/269] CamemBERT & ALBERT doc

---
 docs/source/index.rst               |  5 ++
 docs/source/model_doc/albert.rst    | 71 +++++++++++++++++++++++++++++
 docs/source/model_doc/camembert.rst | 50 ++++++++++++++++++++
 3 files changed, 126 insertions(+)
 create mode 100644 docs/source/model_doc/albert.rst
 create mode 100644 docs/source/model_doc/camembert.rst

diff --git a/docs/source/index.rst b/docs/source/index.rst
index 4cd1f48ba8..55ead33b4d 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -47,6 +47,9 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train
 6. `XLM <https://github.com/facebookresearch/XLM>`_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining <https://arxiv.org/abs/1901.07291>`_ by Guillaume Lample and Alexis Conneau.
 7. `RoBERTa <https://github.com/pytorch/fairseq/tree/master/examples/roberta>`_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach <https://arxiv.org/abs/1907.11692>`_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 8. `DistilBERT <https://huggingface.co/transformers/model_doc/distilbert.html>`_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter <https://arxiv.org/abs/1910.01108>`_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 <https://github.com/huggingface/transformers/tree/master/examples/distillation>`_.
+9. `CTRL <https://github.com/pytorch/fairseq/tree/master/examples/ctrl>`_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation <https://www.github.com/salesforce/ctrl>`_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
+10. `CamemBERT <https://huggingface.co/transformers/model_doc/camembert.html>`_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model <https://arxiv.org/abs/1911.03894>`_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot.
+11. `ALBERT <https://github.com/pytorch/fairseq/tree/master/examples/albert>`_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations <https://arxiv.org/abs/1909.11942>`_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.
 
 .. toctree::
     :maxdepth: 2
@@ -89,3 +92,5 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train
     model_doc/roberta
     model_doc/distilbert
     model_doc/ctrl
+    model_doc/camembert
+    model_doc/albert
diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst
new file mode 100644
index 0000000000..cf52f35eb6
--- /dev/null
+++ b/docs/source/model_doc/albert.rst
@@ -0,0 +1,71 @@
+ALBERT
+----------------------------------------------------
+
+``AlbrtConfig``
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AlbertConfig
+    :members:
+
+
+``AlbertTokenizer``
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AlbertTokenizer
+    :members:
+
+
+``AlbertModel``
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AlbertModel
+    :members:
+
+
+``AlbertForMaskedLM``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AlbertForMaskedLM
+    :members:
+
+
+``AlbertForSequenceClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AlbertForSequenceClassification
+    :members:
+
+
+``AlbertForQuestionAnswering``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AlbertForQuestionAnswering
+    :members:
+
+
+``TFAlbertModel``
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAlbertModel
+    :members:
+
+
+``TFBertForPreTraining``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFBertForPreTraining
+    :members:
+
+
+``TFAlbertForMaskedLM``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAlbertForMaskedLM
+    :members:
+
+
+``TFAlbertForSequenceClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAlbertForSequenceClassification
+    :members:
diff --git a/docs/source/model_doc/camembert.rst b/docs/source/model_doc/camembert.rst
new file mode 100644
index 0000000000..82ca9de945
--- /dev/null
+++ b/docs/source/model_doc/camembert.rst
@@ -0,0 +1,50 @@
+CamemBERT
+----------------------------------------------------
+
+``CamembertConfig``
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertConfig
+    :members:
+
+
+``CamembertTokenizer``
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertTokenizer
+    :members:
+
+
+``CamembertModel``
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertModel
+    :members:
+
+
+``CamembertForMaskedLM``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertForMaskedLM
+    :members:
+
+
+``CamembertForSequenceClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertForSequenceClassification
+    :members:
+
+
+``CamembertForMultipleChoice``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertForMultipleChoice
+    :members:
+
+
+``CamembertForTokenClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.CamembertForTokenClassification
+    :members:

From 44b82c777f0a4e173fa9c68de630a63fbac1b946 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 15:15:11 -0500
Subject: [PATCH 245/269] Updated v2.2.0 doc

---
 .circleci/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 73960cbe99..8081153b9f 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0
 deploy_doc "89fd345" v1.2.0
 deploy_doc "fc9faa8" v2.0.0
 deploy_doc "3ddce1d" v2.1.1
-deploy_doc "668aac4" v2.2.0
+deploy_doc "ee4647b" v2.2.0

From cf26a0c85e0520d3e3770c9ad8dce94318ea3440 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 15:40:03 -0500
Subject: [PATCH 246/269] Fix pretrained models table

---
 README.md                         |  1 +
 docs/source/pretrained_models.rst | 14 +++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 67fb80aecf..dd2c865036 100644
--- a/README.md
+++ b/README.md
@@ -134,6 +134,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation).
 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
+11. **[ALBERT](https://github.com/google-research/google-research/tree/master/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.
 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index d017a3be6c..a6bf508d18 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -162,31 +162,31 @@ Here is the full list of the currently provided pretrained models together with
 | ALBERT            | ``albert-base-v1``                                         | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters                                                            |
 |                   |                                                            | | ALBERT base model                                                                                                                   |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-large-v1``                                        | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters                                                           |
 |                   |                                                            | | ALBERT large model                                                                                                                  |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-xlarge-v1``                                       | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters                                                           |
 |                   |                                                            | | ALBERT xlarge model                                                                                                                 |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-xxlarge-v1``                                      | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters                                                           |
 |                   |                                                            | | ALBERT xxlarge model                                                                                                                |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-base-v2``                                         | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters                                                            |
 |                   |                                                            | | ALBERT base model with no dropout, additional training data and longer training                                                     |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-large-v2``                                        | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters                                                           |
 |                   |                                                            | | ALBERT large model with no dropout, additional training data and longer training                                                    |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-xlarge-v2``                                       | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters                                                           |
 |                   |                                                            | | ALBERT xlarge model with no dropout, additional training data and longer training                                                   |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+|                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 |                   | ``albert-xxlarge-v2``                                      | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters                                                           |
 |                   |                                                            | | ALBERT xxlarge model with no dropout, additional training data and longer training                                                  |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |

From ce02550d50f2074d54d58b37cbc9845d9a159818 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 15:47:02 -0500
Subject: [PATCH 247/269] Fix pretrained models table

---
 docs/source/pretrained_models.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst
index a6bf508d18..a312346df7 100644
--- a/docs/source/pretrained_models.rst
+++ b/docs/source/pretrained_models.rst
@@ -190,7 +190,7 @@ Here is the full list of the currently provided pretrained models together with
 |                   | ``albert-xxlarge-v2``                                      | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters                                                           |
 |                   |                                                            | | ALBERT xxlarge model with no dropout, additional training data and longer training                                                  |
 |                   |                                                            | (see `details <https://github.com/google-research/google-research/tree/master/albert>`__)                                             |
-+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 
 
 .. <https://huggingface.co/transformers/examples.html>`__

From cc7968227e08858df4a5c618c739e1a3ca050196 Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Tue, 26 Nov 2019 15:52:25 -0500
Subject: [PATCH 248/269] Updated v2.2.0 doc

---
 .circleci/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 8081153b9f..61b9f90909 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0
 deploy_doc "89fd345" v1.2.0
 deploy_doc "fc9faa8" v2.0.0
 deploy_doc "3ddce1d" v2.1.1
-deploy_doc "ee4647b" v2.2.0
+deploy_doc "ce02550" v2.2.0

From 361620954acf16b27727d763a591257b03f90b5d Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 27 Nov 2019 10:11:37 -0500
Subject: [PATCH 249/269] Remove TFBertForPreTraining from ALBERT doc

---
 docs/source/model_doc/albert.rst | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst
index cf52f35eb6..92970c9328 100644
--- a/docs/source/model_doc/albert.rst
+++ b/docs/source/model_doc/albert.rst
@@ -50,13 +50,6 @@ ALBERT
     :members:
 
 
-``TFBertForPreTraining``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFBertForPreTraining
-    :members:
-
-
 ``TFAlbertForMaskedLM``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 45d767297a78697e4ff75ec3d83bdf35e43e4fff Mon Sep 17 00:00:00 2001
From: Lysandre <lysandre.debut@reseau.eseo.fr>
Date: Wed, 27 Nov 2019 10:12:20 -0500
Subject: [PATCH 250/269] Updated v2.2.0 doc

---
 .circleci/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh
index 61b9f90909..a32581baef 100755
--- a/.circleci/deploy.sh
+++ b/.circleci/deploy.sh
@@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0
 deploy_doc "89fd345" v1.2.0
 deploy_doc "fc9faa8" v2.0.0
 deploy_doc "3ddce1d" v2.1.1
-deploy_doc "ce02550" v2.2.0
+deploy_doc "3616209" v2.2.0

From 88b317739fe56888528c857fc8e90967148a0051 Mon Sep 17 00:00:00 2001
From: root <huangyingbiao@pku.edu.cn>
Date: Wed, 27 Nov 2019 09:58:38 +0000
Subject: [PATCH 251/269] Fix issue: #1962, input's shape seem to cause error
 in 2.2.0 version tf_albert_model

---
 transformers/modeling_tf_albert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py
index ee122205b9..b2bf66f750 100644
--- a/transformers/modeling_tf_albert.py
+++ b/transformers/modeling_tf_albert.py
@@ -645,7 +645,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
         if input_ids is not None and inputs_embeds is not None:
             raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
         elif input_ids is not None:
-            input_shape = input_ids.shape
+            input_shape = tf.shape(input_ids)
         elif inputs_embeds is not None:
             input_shape = inputs_embeds.shape[:-1]
         else:

From de2696f68e20019fef3a5e1b54de10351abb4145 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Louf?= <remilouf@gmail.com>
Date: Tue, 26 Nov 2019 16:30:27 +0100
Subject: [PATCH 252/269] suggest to track repo w/ https rather than ssh

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 136ef8df81..8228dd59d8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -106,7 +106,7 @@ Follow these steps to start contributing:
    ```bash
    $ git clone git@github.com:<your Github handle>/transformers.git
    $ cd transformers
-   $ git remote add upstream git@github.com:huggingface/transformers.git
+   $ git remote add upstream https://github.com/huggingface/transformers.git
    ```
 
 3. Create a new branch to hold your development changes:

From b5d884d25c96069962030b9e4c9dfb4b1f2f6fa0 Mon Sep 17 00:00:00 2001
From: Julien Chaumond <chaumond@gmail.com>
Date: Wed, 27 Nov 2019 11:05:18 -0500
Subject: [PATCH 253/269] Uniformize #1952

---
 README.md          | 2 +-
 examples/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index dd2c865036..dd20b80590 100644
--- a/README.md
+++ b/README.md
@@ -92,7 +92,7 @@ Examples are included in the repository but are not shipped with the library.
 Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps:
 
 ```bash
-git clone git@github.com:huggingface/transformers
+git clone https://github.com/huggingface/transformers
 cd transformers
 pip install [--editable] .
 ```
diff --git a/examples/README.md b/examples/README.md
index 0c57990ea7..e109a12171 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -7,7 +7,7 @@ similar API between the different models.
 To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment:
 
 ```bash
-git clone git@github.com:huggingface/transformers
+git clone https://github.com/huggingface/transformers
 cd transformers
 pip install [--editable] .
 ```

From 71f71ddb3e04aa18aaa1e3633a2881493e67b438 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 29 Oct 2019 11:50:42 -0400
Subject: [PATCH 254/269] run_xnli + utils_xnli

---
 examples/run_xnli.py   | 534 +++++++++++++++++++++++++++++++++++++++++
 examples/utils_xnli.py |  93 +++++++
 2 files changed, 627 insertions(+)
 create mode 100644 examples/run_xnli.py
 create mode 100644 examples/utils_xnli.py

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
new file mode 100644
index 0000000000..ee37296832
--- /dev/null
+++ b/examples/run_xnli.py
@@ -0,0 +1,534 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning multi-lingual models on XNLI (Bert, XLM).
+    Adapted from `examples/run_glue.py`"""
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import glob
+import logging
+import os
+import random
+
+import numpy as np
+import torch
+from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
+                              TensorDataset)
+from torch.utils.data.distributed import DistributedSampler
+
+try:
+    from torch.utils.tensorboard import SummaryWriter
+except:
+    from tensorboardX import SummaryWriter
+
+from tqdm import tqdm, trange
+
+from transformers import (WEIGHTS_NAME, 
+                          BertConfig, BertForSequenceClassification, BertTokenizer,
+                          XLMConfig, XLMForSequenceClassification, XLMTokenizer,
+                          DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
+
+from transformers import AdamW, WarmupLinearSchedule
+
+from utils_xnli import xnli_compute_metrics as compute_metrics
+from utils_xnli import xnli_output_modes as output_modes
+from utils_xnli import xnli_processors as processors
+
+from transformers import glue_convert_examples_to_features as convert_examples_to_features
+
+logger = logging.getLogger(__name__)
+
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ())
+
+MODEL_CLASSES = {
+    'bert': (BertConfig, BertForSequenceClassification, BertTokenizer),
+    'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
+    # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
+}
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter()
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
+        ]
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
+                                                          output_device=args.local_rank,
+                                                          find_unused_parameters=True)
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
+                   args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1))
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {'input_ids':      batch[0],
+                      'attention_mask': batch[1],
+                      'labels':         batch[3]}
+            if args.model_type != 'distilbert':
+                inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None  # XLM and DistilBERT don't use segment_ids
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean() # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    if args.local_rank == -1 and args.evaluate_during_training:  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                        for key, value in results.items():
+                            tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
+                    tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step)
+                    logging_loss = tr_loss
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.tpu:
+                args.xla_model.optimizer_step(optimizer, barrier=True)
+                model.zero_grad()
+                global_step += 1
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    eval_task_names = (args.task_name,)
+    eval_outputs_dirs = (args.output_dir,)
+
+    results = {}
+    for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
+        eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)
+
+        if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(eval_output_dir)
+
+        args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+        # Note that DistributedSampler samples randomly
+        eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
+        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+        # Eval!
+        logger.info("***** Running evaluation {} *****".format(prefix))
+        logger.info("  Num examples = %d", len(eval_dataset))
+        logger.info("  Batch size = %d", args.eval_batch_size)
+        eval_loss = 0.0
+        nb_eval_steps = 0
+        preds = None
+        out_label_ids = None
+        for batch in tqdm(eval_dataloader, desc="Evaluating"):
+            model.eval()
+            batch = tuple(t.to(args.device) for t in batch)
+
+            with torch.no_grad():
+                inputs = {'input_ids':      batch[0],
+                          'attention_mask': batch[1],
+                          'labels':         batch[3]}
+                if args.model_type != 'distilbert':
+                    inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None  # XLM and DistilBERT don't use segment_ids
+                outputs = model(**inputs)
+                tmp_eval_loss, logits = outputs[:2]
+
+                eval_loss += tmp_eval_loss.mean().item()
+            nb_eval_steps += 1
+            if preds is None:
+                preds = logits.detach().cpu().numpy()
+                out_label_ids = inputs['labels'].detach().cpu().numpy()
+            else:
+                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+                out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)
+
+        eval_loss = eval_loss / nb_eval_steps
+        if args.output_mode == "classification":
+            preds = np.argmax(preds, axis=1)
+        elif args.output_mode == "regression":
+            preds = np.squeeze(preds)
+        result = compute_metrics(eval_task, preds, out_label_ids)
+        results.update(result)
+
+        output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt")
+        with open(output_eval_file, "w") as writer:
+            logger.info("***** Eval results {} *****".format(prefix))
+            for key in sorted(result.keys()):
+                logger.info("  %s = %s", key, str(result[key]))
+                writer.write("%s = %s\n" % (key, str(result[key])))
+
+    return results
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    processor = processors[task](language=args.language, train_language=args.train_language)
+    output_mode = output_modes[task]
+    # Load data features from cache or dataset file
+    cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format(
+        'dev' if evaluate else 'train',
+        list(filter(None, args.model_name_or_path.split('/'))).pop(),
+        str(args.max_seq_length),
+        str(task),
+        str(args.train_language if (not evaluate and args.train_language is not None) else args.language)))
+    if os.path.exists(cached_features_file) and not args.overwrite_cache:
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", args.data_dir)
+        label_list = processor.get_labels()
+        examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
+        features = convert_examples_to_features(examples,
+                                                tokenizer,
+                                                label_list=label_list,
+                                                max_length=args.max_seq_length,
+                                                output_mode=output_mode,
+                                                pad_on_left=False,
+                                                pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
+                                                pad_token_segment_id=0,
+        )
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
+    all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)
+    if output_mode == "classification":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
+    else:
+        raise ValueError(f'No other `output_mode` for XNLI.')
+
+    dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
+    return dataset
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    ## Required parameters
+    parser.add_argument("--data_dir", default=None, type=str, required=True,
+                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
+    parser.add_argument("--model_type", default=None, type=str, required=True,
+                        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
+                        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS))
+    parser.add_argument("--language", default=None, type=str, required=True,
+                        help="Evaluation language. Also train language if `train_language` is set to None.")
+    parser.add_argument("--train_language", default=None, type=str,
+                        help="Train language if is different of the evaluation language.")
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+
+    ## Other parameters
+    parser.add_argument("--config_name", default="", type=str,
+                        help="Pretrained config name or path if not the same as model_name")
+    parser.add_argument("--tokenizer_name", default="", type=str,
+                        help="Pretrained tokenizer name or path if not the same as model_name")
+    parser.add_argument("--cache_dir", default="", type=str,
+                        help="Where do you want to store the pre-trained models downloaded from s3")
+    parser.add_argument("--max_seq_length", default=128, type=int,
+                        help="The maximum total input sequence length after tokenization. Sequences longer "
+                             "than this will be truncated, sequences shorter will be padded.")
+    parser.add_argument("--do_train", action='store_true',
+                        help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true',
+                        help="Whether to run eval on the dev set.")
+    parser.add_argument("--evaluate_during_training", action='store_true',
+                        help="Rul evaluation during training at each logging step.")
+    parser.add_argument("--do_lower_case", action='store_true',
+                        help="Set this flag if you are using an uncased model.")
+
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for evaluation.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument("--learning_rate", default=5e-5, type=float,
+                        help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float,
+                        help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
+                        help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float,
+                        help="Max gradient norm.")
+    parser.add_argument("--num_train_epochs", default=3.0, type=float,
+                        help="Total number of training epochs to perform.")
+    parser.add_argument("--max_steps", default=-1, type=int,
+                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
+    parser.add_argument("--warmup_steps", default=0, type=int,
+                        help="Linear warmup over warmup_steps.")
+
+    parser.add_argument('--logging_steps', type=int, default=50,
+                        help="Log every X updates steps.")
+    parser.add_argument('--save_steps', type=int, default=50,
+                        help="Save checkpoint every X updates steps.")
+    parser.add_argument("--eval_all_checkpoints", action='store_true',
+                        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
+    parser.add_argument("--no_cuda", action='store_true',
+                        help="Avoid using CUDA when available")
+    parser.add_argument('--overwrite_output_dir', action='store_true',
+                        help="Overwrite the content of the output directory")
+    parser.add_argument('--overwrite_cache', action='store_true',
+                        help="Overwrite the cached training and evaluation sets")
+    parser.add_argument('--seed', type=int, default=42,
+                        help="random seed for initialization")
+
+    parser.add_argument('--tpu', action='store_true',
+                        help="Whether to run on the TPU defined in the environment variables")
+    parser.add_argument('--tpu_ip_address', type=str, default='',
+                        help="TPU IP address if none are set in the environment variables")
+    parser.add_argument('--tpu_name', type=str, default='',
+                        help="TPU name if none are set in the environment variables")
+    parser.add_argument('--xrt_tpu_config', type=str, default='',
+                        help="XRT TPU config if none are set in the environment variables")
+
+    parser.add_argument('--fp16', action='store_true',
+                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
+    parser.add_argument('--fp16_opt_level', type=str, default='O1',
+                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                             "See details at https://nvidia.github.io/apex/amp.html")
+    parser.add_argument("--local_rank", type=int, default=-1,
+                        help="For distributed training: local_rank")
+    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
+    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")
+    args = parser.parse_args()
+
+    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
+        raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl')
+        args.n_gpu = 1
+    args.device = device
+
+    if args.tpu:
+        if args.tpu_ip_address:
+            os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address
+        if args.tpu_name:
+            os.environ["TPU_NAME"] = args.tpu_name
+        if args.xrt_tpu_config:
+            os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config
+
+        assert "TPU_IP_ADDRESS" in os.environ
+        assert "TPU_NAME" in os.environ
+        assert "XRT_TPU_CONFIG" in os.environ
+
+        import torch_xla
+        import torch_xla.core.xla_model as xm
+        args.device = xm.xla_device()
+        args.xla_model = xm
+
+    # Setup logging
+    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                        datefmt = '%m/%d/%Y %H:%M:%S',
+                        level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
+    logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+                    args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
+
+    # Set seed
+    set_seed(args)
+
+    # Prepare XNLI task
+    args.task_name = 'xnli'
+    if args.task_name not in processors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+    processor = processors[args.task_name](language=args.language, train_language=args.train_language)
+    args.output_mode = output_modes[args.task_name]
+    label_list = processor.get_labels()
+    num_labels = len(label_list)
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
+    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+
+    # Training
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+
+    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu:
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model.to(args.device)
+
+
+    # Evaluation
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))
+            logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
+            prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else ""
+            
+            model = model_class.from_pretrained(checkpoint)
+            model.to(args.device)
+            result = evaluate(args, model, tokenizer, prefix=prefix)
+            result = dict((k + '_{}'.format(global_step), v) for k, v in result.items())
+            results.update(result)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py
new file mode 100644
index 0000000000..f0238f4664
--- /dev/null
+++ b/examples/utils_xnli.py
@@ -0,0 +1,93 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" XNLI utils (dataset loading and evaluation) """
+
+from __future__ import absolute_import, division, print_function
+
+import logging
+import os
+
+from transformers.data.processors import DataProcessor, InputExample
+from transformers.data.metrics import simple_accuracy
+
+logger = logging.getLogger(__name__)
+
+class XnliProcessor(DataProcessor):
+    """Processor for the XNLI dataset.
+    Adapted from https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/run_classifier.py#L207"""
+
+    def __init__(self, language, train_language = None):
+        self.language = language
+        self.train_language = train_language
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        lg = self.language if self.train_language is None else self.train_language
+        lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv"))
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "%s-%s" % ('train', i)
+            text_a = line[0]
+            text_b = line[1]
+            label = "contradiction" if line[2] == "contradictory" else line[2]
+            assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str)
+            examples.append(
+                InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv"))
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            language = line[0]
+            if language != self.language:
+                continue
+            guid = "%s-%s" % ('dev', i)
+            text_a = line[6]
+            text_b = line[7]
+            label = line[1]
+            assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str)
+            examples.append(
+                InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+    def get_labels(self):
+        """See base class."""
+        return ["contradiction", "entailment", "neutral"]
+
+def xnli_compute_metrics(task_name, preds, labels):
+    assert len(preds) == len(labels)
+    if task_name == "xnli":
+        return {"acc": simple_accuracy(preds, labels)}
+    else:
+        raise ValueError(f'{task_name} is not a supported task.')
+
+xnli_processors = {
+    "xnli": XnliProcessor,
+}
+
+xnli_output_modes = {
+    "xnli": "classification",
+}
+
+xnli_tasks_num_labels = {
+    "xnli": 3,
+}

From d52e98ff9af4509bb803641f0c0d81d67ce73cc3 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 29 Oct 2019 11:51:15 -0400
Subject: [PATCH 255/269] add xnli examples/README.md

---
 examples/README.md | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/examples/README.md b/examples/README.md
index e109a12171..6f8d6bd26e 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -21,6 +21,7 @@ pip install [--editable] .
 | [SQuAD](#squad) | Using BERT/RoBERTa/XLNet/XLM for question answering, examples with distributed training.                                                                                  |
 | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. 
 | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training.                                                                                  |
+| [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. |
 | [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. |
 
 ## TensorFlow 2.0 Bert models on GLUE
@@ -600,3 +601,42 @@ python run_summarization_finetuning.py \
     --do_train \
     --data_path=$DATA_PATH \
 ```
+
+## XNLI
+
+Based on the script [`run_xnli.py`](TODO).
+
+[XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili).
+
+#### Fine-tuning on XNLI
+
+This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min
+on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a 
+`$XNLI_DIR` directory.
+
+* [XNLI 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip)
+* [XNLI-MT 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-MT-1.0.zip)
+
+```bash
+export XNLI_DIR=/path/to/XNLI
+
+python run_xnli.py \
+  --model_type bert \
+  --model_name_or_path bert-base-multilingual-cased \
+  --language en \
+  --train_language en \
+  --do_train \
+  --do_eval \
+  --data_dir $SQUAD_DIR \
+  --per_gpu_train_batch_size 32 \
+  --learning_rate 5e-5 \
+  --num_train_epochs 2.0 \
+  --max_seq_length 128 \
+  --output_dir /tmp/debug_xnli/
+```
+
+Training with the previously defined hyper-parameters yields the following results:
+
+```bash
+TODO
+```

From c4336ecbbdbb8bddcbfb31e0fcc4d382b430a9a5 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 29 Oct 2019 12:04:20 -0400
Subject: [PATCH 256/269] xnli - output_mode consistency

---
 examples/run_xnli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index ee37296832..a9b2e46c13 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -247,8 +247,8 @@ def evaluate(args, model, tokenizer, prefix=""):
         eval_loss = eval_loss / nb_eval_steps
         if args.output_mode == "classification":
             preds = np.argmax(preds, axis=1)
-        elif args.output_mode == "regression":
-            preds = np.squeeze(preds)
+        else:
+            raise ValueError(f'No other `output_mode` for XNLI.')
         result = compute_metrics(eval_task, preds, out_label_ids)
         results.update(result)
 

From 84a0b522cf1b3f69bfbc92eb2309ba5ff652e521 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 29 Oct 2019 18:53:45 -0400
Subject: [PATCH 257/269] mbert reproducibility results

---
 examples/README.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index 6f8d6bd26e..3e5fd03c45 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -604,13 +604,13 @@ python run_summarization_finetuning.py \
 
 ## XNLI
 
-Based on the script [`run_xnli.py`](TODO).
+Based on the script [`run_xnli.py`](https://github.com/huggingface/transformers/blob/master/examples/run_xnli.py).
 
 [XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili).
 
 #### Fine-tuning on XNLI
 
-This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min
+This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in 106 mins
 on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a 
 `$XNLI_DIR` directory.
 
@@ -623,20 +623,21 @@ export XNLI_DIR=/path/to/XNLI
 python run_xnli.py \
   --model_type bert \
   --model_name_or_path bert-base-multilingual-cased \
-  --language en \
+  --language es \
   --train_language en \
   --do_train \
   --do_eval \
-  --data_dir $SQUAD_DIR \
+  --data_dir $XNLI_DIR \
   --per_gpu_train_batch_size 32 \
   --learning_rate 5e-5 \
   --num_train_epochs 2.0 \
   --max_seq_length 128 \
-  --output_dir /tmp/debug_xnli/
+  --output_dir /tmp/debug_xnli/ \
+  --save_steps -1
 ```
 
-Training with the previously defined hyper-parameters yields the following results:
+Training with the previously defined hyper-parameters yields the following results on the dev set:
 
 ```bash
-TODO
+acc = 0.738152610441767
 ```

From cb7b77a8a2ee52812c4358817a6a586f19687cda Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Wed, 30 Oct 2019 18:13:52 -0400
Subject: [PATCH 258/269] fix some typos

---
 transformers/tokenization_xlm.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py
index 01f8721d98..ba994dc356 100644
--- a/transformers/tokenization_xlm.py
+++ b/transformers/tokenization_xlm.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Tokenization classes for OpenAI GPT."""
+"""Tokenization classes for XLM."""
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
@@ -758,9 +758,9 @@ class XLMTokenizer(PreTrainedTokenizer):
         """
         Build model inputs from a sequence or a pair of sequence for sequence classification tasks
         by concatenating and adding special tokens.
-        A RoBERTa sequence has the following format:
+        A XLM sequence has the following format:
             single sequence: <s> X </s>
-            pair of sequences: <s> A </s></s> B </s>
+            pair of sequences: <s> A </s> B </s>
         """
         if token_ids_1 is None:
             return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]

From 289cf4d2b7cf090942e2e7dc4e6134a81095adf6 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 10:10:02 -0500
Subject: [PATCH 259/269] change default for XNLI: dev --> test

---
 examples/run_xnli.py   | 6 +++---
 examples/utils_xnli.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index a9b2e46c13..7fbbf7d152 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -270,7 +270,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
     output_mode = output_modes[task]
     # Load data features from cache or dataset file
     cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format(
-        'dev' if evaluate else 'train',
+        'test' if evaluate else 'train',
         list(filter(None, args.model_name_or_path.split('/'))).pop(),
         str(args.max_seq_length),
         str(task),
@@ -281,7 +281,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
     else:
         logger.info("Creating features from dataset file at %s", args.data_dir)
         label_list = processor.get_labels()
-        examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
+        examples = processor.get_test_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
         features = convert_examples_to_features(examples,
                                                 tokenizer,
                                                 label_list=label_list,
@@ -341,7 +341,7 @@ def main():
     parser.add_argument("--do_train", action='store_true',
                         help="Whether to run training.")
     parser.add_argument("--do_eval", action='store_true',
-                        help="Whether to run eval on the dev set.")
+                        help="Whether to run eval on the test set.")
     parser.add_argument("--evaluate_during_training", action='store_true',
                         help="Rul evaluation during training at each logging step.")
     parser.add_argument("--do_lower_case", action='store_true',
diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py
index f0238f4664..482e79a81f 100644
--- a/examples/utils_xnli.py
+++ b/examples/utils_xnli.py
@@ -50,9 +50,9 @@ class XnliProcessor(DataProcessor):
                 InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
         return examples
 
-    def get_dev_examples(self, data_dir):
+    def get_test_examples(self, data_dir):
         """See base class."""
-        lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv"))
+        lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv"))
         examples = []
         for (i, line) in enumerate(lines):
             if i == 0:
@@ -60,7 +60,7 @@ class XnliProcessor(DataProcessor):
             language = line[0]
             if language != self.language:
                 continue
-            guid = "%s-%s" % ('dev', i)
+            guid = "%s-%s" % ('test', i)
             text_a = line[6]
             text_b = line[7]
             label = line[1]

From d5910b312fbe02a4b08097646e46399dac179084 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 10:21:25 -0500
Subject: [PATCH 260/269] move xnli processor (and utils) to
 transformers/data/processors

---
 examples/run_xnli.py                                        | 6 +++---
 transformers/__init__.py                                    | 3 ++-
 transformers/data/__init__.py                               | 1 +
 transformers/data/processors/__init__.py                    | 2 +-
 .../utils_xnli.py => transformers/data/processors/xnli.py   | 2 +-
 5 files changed, 8 insertions(+), 6 deletions(-)
 rename examples/utils_xnli.py => transformers/data/processors/xnli.py (97%)

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index 7fbbf7d152..0c1743e6c4 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -44,9 +44,9 @@ from transformers import (WEIGHTS_NAME,
 
 from transformers import AdamW, WarmupLinearSchedule
 
-from utils_xnli import xnli_compute_metrics as compute_metrics
-from utils_xnli import xnli_output_modes as output_modes
-from utils_xnli import xnli_processors as processors
+from transformers import xnli_compute_metrics as compute_metrics
+from transformers import xnli_output_modes as output_modes
+from transformers import xnli_processors as processors
 
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 
diff --git a/transformers/__init__.py b/transformers/__init__.py
index dff2479d4b..a133425a9c 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -25,7 +25,8 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH
 from .data import (is_sklearn_available,
                    InputExample, InputFeatures, DataProcessor,
                    glue_output_modes, glue_convert_examples_to_features,
-                   glue_processors, glue_tasks_num_labels)
+                   glue_processors, glue_tasks_num_labels,
+                   xnli_output_modes, xnli_processors, xnli_tasks_num_labels)
 
 if is_sklearn_available():
     from .data import glue_compute_metrics
diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py
index e910d6da2e..46615608a4 100644
--- a/transformers/data/__init__.py
+++ b/transformers/data/__init__.py
@@ -1,5 +1,6 @@
 from .processors import InputExample, InputFeatures, DataProcessor
 from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features
+from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_labels
 
 from .metrics import is_sklearn_available
 if is_sklearn_available():
diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py
index af38c54beb..1c41553ba4 100644
--- a/transformers/data/processors/__init__.py
+++ b/transformers/data/processors/__init__.py
@@ -1,3 +1,3 @@
 from .utils import InputExample, InputFeatures, DataProcessor
 from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features
-
+from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels
diff --git a/examples/utils_xnli.py b/transformers/data/processors/xnli.py
similarity index 97%
rename from examples/utils_xnli.py
rename to transformers/data/processors/xnli.py
index 482e79a81f..96fac761a9 100644
--- a/examples/utils_xnli.py
+++ b/transformers/data/processors/xnli.py
@@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function
 import logging
 import os
 
-from transformers.data.processors import DataProcessor, InputExample
+from .utils import DataProcessor, InputExample
 from transformers.data.metrics import simple_accuracy
 
 logger = logging.getLogger(__name__)

From d75d49a51de28df48b39280b2f06df78d3fa04bf Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 10:32:20 -0500
Subject: [PATCH 261/269] add XnliProcessor to doc

---
 docs/source/main_classes/processors.rst | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst
index a85c126956..a093e621ad 100644
--- a/docs/source/main_classes/processors.rst
+++ b/docs/source/main_classes/processors.rst
@@ -55,4 +55,27 @@ Example usage
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 An example using these processors is given in the
-`run_glue.py <https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_glue.py>`__ script.
\ No newline at end of file
+`run_glue.py <https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_glue.py>`__ script.
+
+
+XNLI
+~~~~~~~~~~~~~~~~~~~~~
+
+`The Cross-Lingual NLI Corpus (XNLI) <https://www.nyu.edu/projects/bowman/xnli/>`__ is a benchmark that evaluates
+the quality of cross-lingual text representations. 
+XNLI is crowd-sourced dataset based on `MultiNLI <http://www.nyu.edu/projects/bowman/multinli/>`: pairs of text are labeled with textual entailment 
+annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili).
+
+It was released together with the paper
+`XNLI: Evaluating Cross-lingual Sentence Representations <https://arxiv.org/abs/1809.05053>`__
+
+This library hosts the processor to load the XNLI data:
+    - :class:`~transformers.data.processors.utils.XnliProcessor`
+
+Please note that since the gold labels are available on the test set, evaluation is performed on the test set.
+
+Example usage
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+An example using these processors is given in the
+`run_xnli.py <https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_xnli.py>`__ script.
\ No newline at end of file

From abd397e95483cc7486bed47fd271ab417e1a288e Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 10:57:24 -0500
Subject: [PATCH 262/269] uniformize w/ the cache_dir update

---
 examples/run_xnli.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index 0c1743e6c4..e8457e542b 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -467,9 +467,17 @@ def main():
 
     args.model_type = args.model_type.lower()
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name)
-    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
-    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          num_labels=num_labels,
+                                          finetuning_task=args.task_name,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
 
     if args.local_rank == 0:
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

From 3e7656f7ac369de08a2ebac3cd1d6870e60605bc Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 11:58:53 -0500
Subject: [PATCH 263/269] update readme

---
 examples/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index 3e5fd03c45..960b218f11 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -623,7 +623,7 @@ export XNLI_DIR=/path/to/XNLI
 python run_xnli.py \
   --model_type bert \
   --model_name_or_path bert-base-multilingual-cased \
-  --language es \
+  --language de \
   --train_language en \
   --do_train \
   --do_eval \
@@ -636,8 +636,8 @@ python run_xnli.py \
   --save_steps -1
 ```
 
-Training with the previously defined hyper-parameters yields the following results on the dev set:
+Training with the previously defined hyper-parameters yields the following results on the **test** set:
 
 ```bash
-acc = 0.738152610441767
+acc = 0.7093812375249501
 ```

From 73fe2e7385af4c6062f366825af570d44cd22fd8 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 12:51:43 -0500
Subject: [PATCH 264/269] remove fstrings

---
 examples/run_xnli.py                 | 4 ++--
 transformers/data/processors/xnli.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index e8457e542b..952128f4ad 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -248,7 +248,7 @@ def evaluate(args, model, tokenizer, prefix=""):
         if args.output_mode == "classification":
             preds = np.argmax(preds, axis=1)
         else:
-            raise ValueError(f'No other `output_mode` for XNLI.')
+            raise ValueError('No other `output_mode` for XNLI.')
         result = compute_metrics(eval_task, preds, out_label_ids)
         results.update(result)
 
@@ -305,7 +305,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
     if output_mode == "classification":
         all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
     else:
-        raise ValueError(f'No other `output_mode` for XNLI.')
+        raise ValueError('No other `output_mode` for XNLI.')
 
     dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
     return dataset
diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py
index 96fac761a9..a4807dd901 100644
--- a/transformers/data/processors/xnli.py
+++ b/transformers/data/processors/xnli.py
@@ -36,7 +36,7 @@ class XnliProcessor(DataProcessor):
     def get_train_examples(self, data_dir):
         """See base class."""
         lg = self.language if self.train_language is None else self.train_language
-        lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv"))
+        lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg)))
         examples = []
         for (i, line) in enumerate(lines):
             if i == 0:
@@ -78,7 +78,7 @@ def xnli_compute_metrics(task_name, preds, labels):
     if task_name == "xnli":
         return {"acc": simple_accuracy(preds, labels)}
     else:
-        raise ValueError(f'{task_name} is not a supported task.')
+        raise ValueError('{} is not a supported task.'.format(task_name))
 
 xnli_processors = {
     "xnli": XnliProcessor,

From bcd8dc6b48a335b899f00b59f016f740c5230d41 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 12:53:08 -0500
Subject: [PATCH 265/269] move xnli_compute_metrics to data/metrics

---
 transformers/__init__.py              | 2 +-
 transformers/data/__init__.py         | 2 +-
 transformers/data/metrics/__init__.py | 8 ++++++++
 transformers/data/processors/xnli.py  | 7 -------
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/transformers/__init__.py b/transformers/__init__.py
index a133425a9c..b29ad38e73 100644
--- a/transformers/__init__.py
+++ b/transformers/__init__.py
@@ -29,7 +29,7 @@ from .data import (is_sklearn_available,
                    xnli_output_modes, xnli_processors, xnli_tasks_num_labels)
 
 if is_sklearn_available():
-    from .data import glue_compute_metrics
+    from .data import glue_compute_metrics, xnli_compute_metrics
 
 # Tokenizers
 from .tokenization_utils import (PreTrainedTokenizer)
diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py
index 46615608a4..b811a35807 100644
--- a/transformers/data/__init__.py
+++ b/transformers/data/__init__.py
@@ -4,4 +4,4 @@ from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_label
 
 from .metrics import is_sklearn_available
 if is_sklearn_available():
-    from .metrics import glue_compute_metrics
+    from .metrics import glue_compute_metrics, xnli_compute_metrics
diff --git a/transformers/data/metrics/__init__.py b/transformers/data/metrics/__init__.py
index c9ebaac38d..5a46eb05d3 100644
--- a/transformers/data/metrics/__init__.py
+++ b/transformers/data/metrics/__init__.py
@@ -81,3 +81,11 @@ if _has_sklearn:
             return {"acc": simple_accuracy(preds, labels)}
         else:
             raise KeyError(task_name)
+
+
+    def xnli_compute_metrics(task_name, preds, labels):
+        assert len(preds) == len(labels)
+        if task_name == "xnli":
+            return {"acc": simple_accuracy(preds, labels)}
+        else:
+            raise KeyError(task_name)
diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py
index a4807dd901..ce582f31a6 100644
--- a/transformers/data/processors/xnli.py
+++ b/transformers/data/processors/xnli.py
@@ -73,13 +73,6 @@ class XnliProcessor(DataProcessor):
         """See base class."""
         return ["contradiction", "entailment", "neutral"]
 
-def xnli_compute_metrics(task_name, preds, labels):
-    assert len(preds) == len(labels)
-    if task_name == "xnli":
-        return {"acc": simple_accuracy(preds, labels)}
-    else:
-        raise ValueError('{} is not a supported task.'.format(task_name))
-
 xnli_processors = {
     "xnli": XnliProcessor,
 }

From d47402263964e30ee17cbc06811622bf2df50d6d Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 12:56:03 -0500
Subject: [PATCH 266/269] cleaning simple_accuracy since not used anymore

---
 transformers/data/processors/xnli.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py
index ce582f31a6..efbe3762f6 100644
--- a/transformers/data/processors/xnli.py
+++ b/transformers/data/processors/xnli.py
@@ -21,7 +21,6 @@ import logging
 import os
 
 from .utils import DataProcessor, InputExample
-from transformers.data.metrics import simple_accuracy
 
 logger = logging.getLogger(__name__)
 

From 07ab8d7af6041f0d3562badcb3d6f173062329a1 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Tue, 5 Nov 2019 17:33:14 -0500
Subject: [PATCH 267/269] fix bug

---
 transformers/data/processors/xnli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py
index efbe3762f6..958bdf62f9 100644
--- a/transformers/data/processors/xnli.py
+++ b/transformers/data/processors/xnli.py
@@ -35,7 +35,7 @@ class XnliProcessor(DataProcessor):
     def get_train_examples(self, data_dir):
         """See base class."""
         lg = self.language if self.train_language is None else self.train_language
-        lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg)))
+        lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{}.tsv".format(lg)))
         examples = []
         for (i, line) in enumerate(lines):
             if i == 0:

From d5478b939d64db58972e46b7218c765c918b76ac Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Mon, 25 Nov 2019 19:40:48 +0000
Subject: [PATCH 268/269] add distilbert + update run_xnli wrt run_glue

---
 examples/run_xnli.py | 51 +++++++++++---------------------------------
 1 file changed, 12 insertions(+), 39 deletions(-)

diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index 952128f4ad..a3bc0d4604 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Finetuning multi-lingual models on XNLI (Bert, XLM).
+""" Finetuning multi-lingual models on XNLI (Bert, DistilBERT, XLM).
     Adapted from `examples/run_glue.py`"""
 
 from __future__ import absolute_import, division, print_function
@@ -42,7 +42,7 @@ from transformers import (WEIGHTS_NAME,
                           XLMConfig, XLMForSequenceClassification, XLMTokenizer,
                           DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
 
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW, get_linear_schedule_with_warmup
 
 from transformers import xnli_compute_metrics as compute_metrics
 from transformers import xnli_output_modes as output_modes
@@ -52,12 +52,12 @@ from transformers import glue_convert_examples_to_features as convert_examples_t
 
 logger = logging.getLogger(__name__)
 
-ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ())
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, DistilBertConfig, XLMConfig)), ())
 
 MODEL_CLASSES = {
     'bert': (BertConfig, BertForSequenceClassification, BertTokenizer),
     'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
-    # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
+    'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
 }
 
 
@@ -91,7 +91,7 @@ def train(args, train_dataset, model, tokenizer):
         {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
         ]
     optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
-    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
     if args.fp16:
         try:
             from apex import amp
@@ -149,7 +149,7 @@ def train(args, train_dataset, model, tokenizer):
                 loss.backward()
 
             tr_loss += loss.item()
-            if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu:
+            if (step + 1) % args.gradient_accumulation_steps == 0:
                 if args.fp16:
                     torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
                 else:
@@ -180,11 +180,6 @@ def train(args, train_dataset, model, tokenizer):
                     torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                     logger.info("Saving model checkpoint to %s", output_dir)
 
-            if args.tpu:
-                args.xla_model.optimizer_step(optimizer, barrier=True)
-                model.zero_grad()
-                global_step += 1
-
             if args.max_steps > 0 and global_step > args.max_steps:
                 epoch_iterator.close()
                 break
@@ -214,6 +209,10 @@ def evaluate(args, model, tokenizer, prefix=""):
         eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
+        # multi-gpu eval
+        if args.n_gpu > 1:
+            model = torch.nn.DataParallel(model)
+
         # Eval!
         logger.info("***** Running evaluation {} *****".format(prefix))
         logger.info("  Num examples = %d", len(eval_dataset))
@@ -383,15 +382,6 @@ def main():
     parser.add_argument('--seed', type=int, default=42,
                         help="random seed for initialization")
 
-    parser.add_argument('--tpu', action='store_true',
-                        help="Whether to run on the TPU defined in the environment variables")
-    parser.add_argument('--tpu_ip_address', type=str, default='',
-                        help="TPU IP address if none are set in the environment variables")
-    parser.add_argument('--tpu_name', type=str, default='',
-                        help="TPU name if none are set in the environment variables")
-    parser.add_argument('--xrt_tpu_config', type=str, default='',
-                        help="XRT TPU config if none are set in the environment variables")
-
     parser.add_argument('--fp16', action='store_true',
                         help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
     parser.add_argument('--fp16_opt_level', type=str, default='O1',
@@ -425,23 +415,6 @@ def main():
         args.n_gpu = 1
     args.device = device
 
-    if args.tpu:
-        if args.tpu_ip_address:
-            os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address
-        if args.tpu_name:
-            os.environ["TPU_NAME"] = args.tpu_name
-        if args.xrt_tpu_config:
-            os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config
-
-        assert "TPU_IP_ADDRESS" in os.environ
-        assert "TPU_NAME" in os.environ
-        assert "XRT_TPU_CONFIG" in os.environ
-
-        import torch_xla
-        import torch_xla.core.xla_model as xm
-        args.device = xm.xla_device()
-        args.xla_model = xm
-
     # Setup logging
     logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                         datefmt = '%m/%d/%Y %H:%M:%S',
@@ -495,7 +468,7 @@ def main():
 
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
-    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu:
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
         # Create output directory if needed
         if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
             os.makedirs(args.output_dir)
@@ -512,7 +485,7 @@ def main():
 
         # Load a trained model and vocabulary that you have fine-tuned
         model = model_class.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
         model.to(args.device)
 
 

From 10bd1ddb39235b2f58594e48867595e7d38cd619 Mon Sep 17 00:00:00 2001
From: VictorSanh <victorsanh@gmail.com>
Date: Mon, 25 Nov 2019 19:41:00 +0000
Subject: [PATCH 269/269] soft launch distilbert multilingual

---
 transformers/configuration_distilbert.py | 3 ++-
 transformers/modeling_distilbert.py      | 3 ++-
 transformers/tokenization_distilbert.py  | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py
index 2a8a149acf..f929a9bc39 100644
--- a/transformers/configuration_distilbert.py
+++ b/transformers/configuration_distilbert.py
@@ -27,7 +27,8 @@ logger = logging.getLogger(__name__)
 
 DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json",
-    'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json"
+    'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json",
+    'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-config.json",
 }
 
 
diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py
index d30f493c69..62c623ff6c 100644
--- a/transformers/modeling_distilbert.py
+++ b/transformers/modeling_distilbert.py
@@ -42,7 +42,8 @@ logger = logging.getLogger(__name__)
 
 DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
     'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin",
-    'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin"
+    'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin",
+    'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-pytorch_model.bin",
 }
 
 
diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py
index dfa02926d8..832f0c3d0b 100644
--- a/transformers/tokenization_distilbert.py
+++ b/transformers/tokenization_distilbert.py
@@ -33,12 +33,14 @@ PRETRAINED_VOCAB_FILES_MAP = {
     {
         'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
         'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt",
+        'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt",
     }
 }
 
 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
     'distilbert-base-uncased': 512,
     'distilbert-base-uncased-distilled-squad': 512,
+    'distilbert-base-multilingual-cased': 512,
 }